2024-11-08 09:22:20,339 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 09:22:20,350 main DEBUG Took 0.009145 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 09:22:20,351 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 09:22:20,351 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 09:22:20,352 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 09:22:20,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,359 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 09:22:20,372 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,374 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,374 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,375 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,376 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,377 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,378 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,379 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,380 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,381 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,382 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,383 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,383 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 09:22:20,384 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,384 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 09:22:20,386 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 09:22:20,387 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 09:22:20,390 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 09:22:20,390 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 09:22:20,392 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 09:22:20,392 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 09:22:20,400 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 09:22:20,402 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 09:22:20,404 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 09:22:20,404 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 09:22:20,405 main DEBUG createAppenders(={Console}) 2024-11-08 09:22:20,406 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-08 09:22:20,406 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 09:22:20,406 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-08 09:22:20,407 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 09:22:20,407 main DEBUG OutputStream closed 2024-11-08 09:22:20,407 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 09:22:20,408 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 09:22:20,408 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-08 09:22:20,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 09:22:20,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 09:22:20,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 09:22:20,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 09:22:20,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 09:22:20,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 09:22:20,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 09:22:20,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 09:22:20,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 09:22:20,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 09:22:20,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 09:22:20,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 09:22:20,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 09:22:20,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 09:22:20,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 09:22:20,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 09:22:20,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 09:22:20,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 09:22:20,509 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 09:22:20,510 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-08 09:22:20,510 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 09:22:20,511 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-08T09:22:20,751 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec 2024-11-08 09:22:20,755 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 09:22:20,756 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T09:22:20,768 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-08T09:22:20,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T09:22:20,819 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f, deleteOnExit=true 2024-11-08T09:22:20,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T09:22:20,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/test.cache.data in system properties and HBase conf 2024-11-08T09:22:20,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T09:22:20,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/hadoop.log.dir in system properties and HBase conf 2024-11-08T09:22:20,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T09:22:20,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T09:22:20,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T09:22:20,920 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T09:22:21,008 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T09:22:21,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T09:22:21,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T09:22:21,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T09:22:21,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T09:22:21,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T09:22:21,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T09:22:21,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T09:22:21,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T09:22:21,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T09:22:21,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/nfs.dump.dir in system properties and HBase conf 2024-11-08T09:22:21,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/java.io.tmpdir in system properties and HBase conf 2024-11-08T09:22:21,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T09:22:21,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T09:22:21,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T09:22:22,133 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T09:22:22,208 INFO [Time-limited test {}] log.Log(170): Logging initialized @2746ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T09:22:22,278 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T09:22:22,341 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T09:22:22,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T09:22:22,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T09:22:22,365 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T09:22:22,377 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T09:22:22,380 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/hadoop.log.dir/,AVAILABLE} 2024-11-08T09:22:22,381 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T09:22:22,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/java.io.tmpdir/jetty-localhost-33729-hadoop-hdfs-3_4_1-tests_jar-_-any-1764191699717804032/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T09:22:22,567 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:33729} 2024-11-08T09:22:22,567 INFO [Time-limited test {}] server.Server(415): Started @3106ms 2024-11-08T09:22:23,266 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T09:22:23,275 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T09:22:23,276 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T09:22:23,277 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T09:22:23,277 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T09:22:23,278 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/hadoop.log.dir/,AVAILABLE} 2024-11-08T09:22:23,278 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T09:22:23,403 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/java.io.tmpdir/jetty-localhost-44531-hadoop-hdfs-3_4_1-tests_jar-_-any-1244791136985499933/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T09:22:23,404 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:44531} 2024-11-08T09:22:23,405 INFO [Time-limited test {}] server.Server(415): Started @3943ms 2024-11-08T09:22:23,456 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T09:22:24,515 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f/data/data2/current/BP-1227818152-172.17.0.2-1731057741605/current, will proceed with Du for space computation calculation, 2024-11-08T09:22:24,515 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f/data/data1/current/BP-1227818152-172.17.0.2-1731057741605/current, will proceed with Du for space computation calculation, 2024-11-08T09:22:24,545 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T09:22:24,586 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6d4a70242b4e7da with lease ID 0xe190af254a07e9b9: Processing first storage report for DS-015ea186-005e-4d45-9996-56003f743725 from datanode DatanodeRegistration(127.0.0.1:34747, datanodeUuid=3f329c0b-3c7d-416d-976d-4b847f832cf6, infoPort=35751, infoSecurePort=0, ipcPort=36197, storageInfo=lv=-57;cid=testClusterID;nsid=1717780772;c=1731057741605) 2024-11-08T09:22:24,588 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6d4a70242b4e7da with lease ID 0xe190af254a07e9b9: from storage DS-015ea186-005e-4d45-9996-56003f743725 node DatanodeRegistration(127.0.0.1:34747, datanodeUuid=3f329c0b-3c7d-416d-976d-4b847f832cf6, infoPort=35751, infoSecurePort=0, ipcPort=36197, storageInfo=lv=-57;cid=testClusterID;nsid=1717780772;c=1731057741605), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T09:22:24,588 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6d4a70242b4e7da with lease ID 0xe190af254a07e9b9: Processing first storage report for DS-40393ccd-ace4-43f2-8534-e1baad571a0a from datanode DatanodeRegistration(127.0.0.1:34747, datanodeUuid=3f329c0b-3c7d-416d-976d-4b847f832cf6, infoPort=35751, infoSecurePort=0, ipcPort=36197, storageInfo=lv=-57;cid=testClusterID;nsid=1717780772;c=1731057741605) 2024-11-08T09:22:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6d4a70242b4e7da with lease ID 0xe190af254a07e9b9: from storage DS-40393ccd-ace4-43f2-8534-e1baad571a0a node DatanodeRegistration(127.0.0.1:34747, datanodeUuid=3f329c0b-3c7d-416d-976d-4b847f832cf6, infoPort=35751, infoSecurePort=0, ipcPort=36197, storageInfo=lv=-57;cid=testClusterID;nsid=1717780772;c=1731057741605), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T09:22:24,656 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec 2024-11-08T09:22:24,721 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f/zookeeper_0, clientPort=57608, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T09:22:24,730 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57608 2024-11-08T09:22:24,741 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T09:22:24,743 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T09:22:24,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741825_1001 (size=7) 2024-11-08T09:22:25,392 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 with version=8 2024-11-08T09:22:25,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/hbase-staging 2024-11-08T09:22:25,498 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T09:22:25,794 INFO [Time-limited test {}] client.ConnectionUtils(128): master/cf4d76213909:0 server-side Connection retries=45 2024-11-08T09:22:25,806 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T09:22:25,806 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T09:22:25,812 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T09:22:25,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T09:22:25,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T09:22:26,018 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T09:22:26,085 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T09:22:26,099 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T09:22:26,105 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T09:22:26,139 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30873 (auto-detected) 2024-11-08T09:22:26,140 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T09:22:26,164 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35807 2024-11-08T09:22:26,193 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35807 connecting to ZooKeeper ensemble=127.0.0.1:57608 2024-11-08T09:22:26,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358070x0, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T09:22:26,321 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35807-0x10119c0f6eb0000 connected 2024-11-08T09:22:26,552 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T09:22:26,555 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T09:22:26,570 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T09:22:26,574 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1, hbase.cluster.distributed=false 2024-11-08T09:22:26,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T09:22:26,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-08T09:22:26,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35807 2024-11-08T09:22:26,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35807 2024-11-08T09:22:26,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-08T09:22:26,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-08T09:22:26,737 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cf4d76213909:0 server-side Connection retries=45 2024-11-08T09:22:26,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T09:22:26,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T09:22:26,740 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T09:22:26,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T09:22:26,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T09:22:26,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T09:22:26,747 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T09:22:26,748 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39151 2024-11-08T09:22:26,751 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39151 connecting to ZooKeeper ensemble=127.0.0.1:57608 2024-11-08T09:22:26,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T09:22:26,757 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T09:22:26,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391510x0, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T09:22:26,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:391510x0, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T09:22:26,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39151-0x10119c0f6eb0001 connected 2024-11-08T09:22:26,794 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T09:22:26,802 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T09:22:26,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T09:22:26,810 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T09:22:26,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39151 2024-11-08T09:22:26,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39151 2024-11-08T09:22:26,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39151 2024-11-08T09:22:26,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39151 2024-11-08T09:22:26,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39151 2024-11-08T09:22:26,831 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cf4d76213909:35807 2024-11-08T09:22:26,832 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cf4d76213909,35807,1731057745600 2024-11-08T09:22:26,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T09:22:26,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T09:22:26,892 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cf4d76213909,35807,1731057745600 2024-11-08T09:22:26,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T09:22:26,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:26,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:26,921 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T09:22:26,923 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cf4d76213909,35807,1731057745600 from backup master directory 2024-11-08T09:22:26,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T09:22:26,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cf4d76213909,35807,1731057745600 2024-11-08T09:22:26,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T09:22:26,938 WARN [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T09:22:26,939 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cf4d76213909,35807,1731057745600 2024-11-08T09:22:26,941 INFO [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T09:22:26,942 INFO [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T09:22:27,004 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/hbase.id] with ID: 59c75356-9176-4dc6-a8b1-6b5efa72c120 2024-11-08T09:22:27,005 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/.tmp/hbase.id 2024-11-08T09:22:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741826_1002 (size=42) 2024-11-08T09:22:27,417 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/.tmp/hbase.id]:[hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/hbase.id] 2024-11-08T09:22:27,461 INFO [master/cf4d76213909:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T09:22:27,466 INFO [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T09:22:27,484 INFO [master/cf4d76213909:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-08T09:22:27,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:27,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741827_1003 (size=196) 2024-11-08T09:22:27,941 INFO [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:22:27,944 INFO [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T09:22:27,964 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:27,970 INFO [master/cf4d76213909:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T09:22:28,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741828_1004 (size=1189) 2024-11-08T09:22:28,049 INFO [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store 2024-11-08T09:22:28,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741829_1005 (size=34) 2024-11-08T09:22:28,496 INFO [master/cf4d76213909:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T09:22:28,499 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:22:28,500 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T09:22:28,500 INFO [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:22:28,500 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:22:28,502 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T09:22:28,502 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:22:28,502 INFO [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:22:28,503 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731057748500Disabling compacts and flushes for region at 1731057748500Disabling writes for close at 1731057748502 (+2 ms)Writing region close event to WAL at 1731057748502Closed at 1731057748502 2024-11-08T09:22:28,506 WARN [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/.initializing 2024-11-08T09:22:28,506 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/WALs/cf4d76213909,35807,1731057745600 2024-11-08T09:22:28,514 INFO [master/cf4d76213909:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T09:22:28,530 INFO [master/cf4d76213909:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C35807%2C1731057745600, suffix=, logDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/WALs/cf4d76213909,35807,1731057745600, archiveDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/oldWALs, maxLogs=10 2024-11-08T09:22:28,555 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/WALs/cf4d76213909,35807,1731057745600/cf4d76213909%2C35807%2C1731057745600.1731057748535, exclude list is [], retry=0 2024-11-08T09:22:28,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34747,DS-015ea186-005e-4d45-9996-56003f743725,DISK] 2024-11-08T09:22:28,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T09:22:28,617 INFO [master/cf4d76213909:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/WALs/cf4d76213909,35807,1731057745600/cf4d76213909%2C35807%2C1731057745600.1731057748535 2024-11-08T09:22:28,618 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35751:35751)] 2024-11-08T09:22:28,619 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:22:28,619 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:22:28,624 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,625 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,702 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T09:22:28,705 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:28,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:28,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T09:22:28,711 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:28,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:22:28,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T09:22:28,716 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:28,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:22:28,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T09:22:28,720 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:28,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:22:28,722 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,727 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,728 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,733 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,734 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,738 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T09:22:28,743 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T09:22:28,748 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:22:28,749 INFO [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60543336, jitterRate=-0.09783399105072021}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T09:22:28,758 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731057748642Initializing all the Stores at 1731057748645 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731057748646 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057748647 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057748647Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057748647Cleaning up temporary data from old regions at 1731057748734 (+87 ms)Region opened successfully at 1731057748757 (+23 ms) 2024-11-08T09:22:28,760 INFO [master/cf4d76213909:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T09:22:28,799 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d774ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cf4d76213909/172.17.0.2:0 2024-11-08T09:22:28,828 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T09:22:28,839 INFO [master/cf4d76213909:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T09:22:28,839 INFO [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T09:22:28,842 INFO [master/cf4d76213909:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T09:22:28,843 INFO [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T09:22:28,848 INFO [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T09:22:28,848 INFO [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T09:22:28,872 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T09:22:28,881 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T09:22:28,930 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T09:22:28,933 INFO [master/cf4d76213909:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T09:22:28,935 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T09:22:28,947 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T09:22:28,950 INFO [master/cf4d76213909:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T09:22:28,953 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T09:22:28,962 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T09:22:28,964 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T09:22:28,972 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T09:22:28,991 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T09:22:29,004 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T09:22:29,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T09:22:29,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T09:22:29,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,018 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cf4d76213909,35807,1731057745600, sessionid=0x10119c0f6eb0000, setting cluster-up flag (Was=false) 2024-11-08T09:22:29,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,078 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T09:22:29,080 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cf4d76213909,35807,1731057745600 2024-11-08T09:22:29,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,141 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T09:22:29,143 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cf4d76213909,35807,1731057745600 2024-11-08T09:22:29,151 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T09:22:29,222 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(746): ClusterId : 59c75356-9176-4dc6-a8b1-6b5efa72c120 2024-11-08T09:22:29,225 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T09:22:29,234 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T09:22:29,243 INFO [master/cf4d76213909:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T09:22:29,249 INFO [master/cf4d76213909:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T09:22:29,254 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cf4d76213909,35807,1731057745600 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T09:22:29,259 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T09:22:29,259 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T09:22:29,261 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5 2024-11-08T09:22:29,261 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5 2024-11-08T09:22:29,261 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5 2024-11-08T09:22:29,262 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cf4d76213909:0, corePoolSize=5, maxPoolSize=5 2024-11-08T09:22:29,262 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cf4d76213909:0, corePoolSize=10, maxPoolSize=10 2024-11-08T09:22:29,262 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,262 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cf4d76213909:0, corePoolSize=2, maxPoolSize=2 2024-11-08T09:22:29,263 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,268 INFO [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731057779267 2024-11-08T09:22:29,269 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T09:22:29,270 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T09:22:29,270 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T09:22:29,271 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T09:22:29,275 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T09:22:29,276 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T09:22:29,276 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T09:22:29,276 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T09:22:29,277 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:29,277 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T09:22:29,279 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,284 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T09:22:29,285 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T09:22:29,286 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T09:22:29,288 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T09:22:29,289 INFO [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T09:22:29,291 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.large.0-1731057749291,5,FailOnTimeoutGroup] 2024-11-08T09:22:29,292 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.small.0-1731057749292,5,FailOnTimeoutGroup] 2024-11-08T09:22:29,292 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,293 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T09:22:29,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741831_1007 (size=1321) 2024-11-08T09:22:29,294 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,296 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,297 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T09:22:29,297 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:22:29,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741832_1008 (size=32) 2024-11-08T09:22:29,313 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:22:29,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T09:22:29,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T09:22:29,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:29,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:29,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T09:22:29,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T09:22:29,329 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:29,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:29,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T09:22:29,335 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T09:22:29,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:29,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:29,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T09:22:29,339 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T09:22:29,340 DEBUG [RS:0;cf4d76213909:39151 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d73fc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cf4d76213909/172.17.0.2:0 2024-11-08T09:22:29,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T09:22:29,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:29,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:29,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T09:22:29,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740 2024-11-08T09:22:29,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740 2024-11-08T09:22:29,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T09:22:29,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T09:22:29,356 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:22:29,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T09:22:29,365 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:22:29,366 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71140771, jitterRate=0.06008009612560272}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:22:29,369 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cf4d76213909:39151 2024-11-08T09:22:29,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731057749313Initializing all the Stores at 1731057749316 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731057749316Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731057749316Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057749316Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731057749316Cleaning up temporary data from old regions at 1731057749354 (+38 ms)Region opened successfully at 1731057749370 (+16 ms) 2024-11-08T09:22:29,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T09:22:29,371 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T09:22:29,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T09:22:29,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T09:22:29,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T09:22:29,373 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T09:22:29,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731057749371Disabling compacts and flushes for region at 1731057749371Disabling writes for close at 1731057749371Writing region close event to WAL at 1731057749372 (+1 ms)Closed at 1731057749373 (+1 ms) 2024-11-08T09:22:29,373 INFO [RS:0;cf4d76213909:39151 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T09:22:29,374 INFO [RS:0;cf4d76213909:39151 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T09:22:29,374 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T09:22:29,377 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(2659): reportForDuty to master=cf4d76213909,35807,1731057745600 with port=39151, startcode=1731057746690 2024-11-08T09:22:29,378 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T09:22:29,378 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T09:22:29,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T09:22:29,393 DEBUG [RS:0;cf4d76213909:39151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T09:22:29,400 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T09:22:29,404 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T09:22:29,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59897, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T09:22:29,445 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cf4d76213909,39151,1731057746690 2024-11-08T09:22:29,448 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35807 {}] master.ServerManager(517): Registering regionserver=cf4d76213909,39151,1731057746690 2024-11-08T09:22:29,466 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:22:29,466 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43921 2024-11-08T09:22:29,466 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T09:22:29,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T09:22:29,513 DEBUG [RS:0;cf4d76213909:39151 {}] zookeeper.ZKUtil(111): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cf4d76213909,39151,1731057746690 2024-11-08T09:22:29,513 WARN [RS:0;cf4d76213909:39151 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T09:22:29,514 INFO [RS:0;cf4d76213909:39151 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T09:22:29,514 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/WALs/cf4d76213909,39151,1731057746690 2024-11-08T09:22:29,515 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cf4d76213909,39151,1731057746690] 2024-11-08T09:22:29,543 INFO [RS:0;cf4d76213909:39151 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T09:22:29,555 WARN [cf4d76213909:35807 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-08T09:22:29,561 INFO [RS:0;cf4d76213909:39151 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T09:22:29,566 INFO [RS:0;cf4d76213909:39151 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T09:22:29,567 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,568 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T09:22:29,574 INFO [RS:0;cf4d76213909:39151 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T09:22:29,576 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,576 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,576 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,577 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,577 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,577 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,577 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cf4d76213909:0, corePoolSize=2, maxPoolSize=2 2024-11-08T09:22:29,578 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,578 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,578 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,578 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,578 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,579 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cf4d76213909:0, corePoolSize=1, maxPoolSize=1 2024-11-08T09:22:29,579 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3 2024-11-08T09:22:29,579 DEBUG [RS:0;cf4d76213909:39151 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0, corePoolSize=3, maxPoolSize=3 2024-11-08T09:22:29,582 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,582 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,582 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,582 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,582 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,582 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,39151,1731057746690-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T09:22:29,602 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T09:22:29,605 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,39151,1731057746690-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,605 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,606 INFO [RS:0;cf4d76213909:39151 {}] regionserver.Replication(171): cf4d76213909,39151,1731057746690 started 2024-11-08T09:22:29,629 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:29,629 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1482): Serving as cf4d76213909,39151,1731057746690, RpcServer on cf4d76213909/172.17.0.2:39151, sessionid=0x10119c0f6eb0001 2024-11-08T09:22:29,630 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T09:22:29,631 DEBUG [RS:0;cf4d76213909:39151 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cf4d76213909,39151,1731057746690 2024-11-08T09:22:29,631 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,39151,1731057746690' 2024-11-08T09:22:29,631 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T09:22:29,632 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T09:22:29,633 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T09:22:29,633 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T09:22:29,633 DEBUG [RS:0;cf4d76213909:39151 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cf4d76213909,39151,1731057746690 2024-11-08T09:22:29,633 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cf4d76213909,39151,1731057746690' 2024-11-08T09:22:29,633 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T09:22:29,634 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T09:22:29,635 DEBUG [RS:0;cf4d76213909:39151 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T09:22:29,635 INFO [RS:0;cf4d76213909:39151 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T09:22:29,635 INFO [RS:0;cf4d76213909:39151 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T09:22:29,740 INFO [RS:0;cf4d76213909:39151 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T09:22:29,743 INFO [RS:0;cf4d76213909:39151 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C39151%2C1731057746690, suffix=, logDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/WALs/cf4d76213909,39151,1731057746690, archiveDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/oldWALs, maxLogs=32 2024-11-08T09:22:29,758 DEBUG [RS:0;cf4d76213909:39151 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/WALs/cf4d76213909,39151,1731057746690/cf4d76213909%2C39151%2C1731057746690.1731057749745, exclude list is [], retry=0 2024-11-08T09:22:29,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34747,DS-015ea186-005e-4d45-9996-56003f743725,DISK] 2024-11-08T09:22:29,766 INFO [RS:0;cf4d76213909:39151 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/WALs/cf4d76213909,39151,1731057746690/cf4d76213909%2C39151%2C1731057746690.1731057749745 2024-11-08T09:22:29,767 DEBUG [RS:0;cf4d76213909:39151 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35751:35751)] 2024-11-08T09:22:29,808 DEBUG [cf4d76213909:35807 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T09:22:29,820 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:22:29,826 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cf4d76213909,39151,1731057746690, state=OPENING 2024-11-08T09:22:29,884 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T09:22:29,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:22:29,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T09:22:29,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T09:22:29,926 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T09:22:29,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:22:30,108 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T09:22:30,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50933, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T09:22:30,122 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T09:22:30,123 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T09:22:30,124 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T09:22:30,128 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cf4d76213909%2C39151%2C1731057746690.meta, suffix=.meta, logDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/WALs/cf4d76213909,39151,1731057746690, archiveDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/oldWALs, maxLogs=32 2024-11-08T09:22:30,147 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/WALs/cf4d76213909,39151,1731057746690/cf4d76213909%2C39151%2C1731057746690.meta.1731057750130.meta, exclude list is [], retry=0 2024-11-08T09:22:30,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34747,DS-015ea186-005e-4d45-9996-56003f743725,DISK] 2024-11-08T09:22:30,154 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/WALs/cf4d76213909,39151,1731057746690/cf4d76213909%2C39151%2C1731057746690.meta.1731057750130.meta 2024-11-08T09:22:30,154 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35751:35751)] 2024-11-08T09:22:30,155 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:22:30,156 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T09:22:30,159 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T09:22:30,162 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T09:22:30,166 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T09:22:30,167 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:22:30,167 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T09:22:30,167 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T09:22:30,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T09:22:30,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T09:22:30,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:30,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:30,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T09:22:30,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T09:22:30,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:30,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:30,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T09:22:30,177 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T09:22:30,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:30,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:30,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T09:22:30,180 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T09:22:30,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:30,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T09:22:30,181 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T09:22:30,183 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740 2024-11-08T09:22:30,185 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740 2024-11-08T09:22:30,188 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T09:22:30,188 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T09:22:30,189 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:22:30,191 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T09:22:30,193 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68724583, jitterRate=0.02407608926296234}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:22:30,193 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T09:22:30,195 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731057750168Writing region info on filesystem at 1731057750168Initializing all the Stores at 1731057750170 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731057750170Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731057750170Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057750170Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731057750170Cleaning up temporary data from old regions at 1731057750188 (+18 ms)Running coprocessor post-open hooks at 1731057750193 (+5 ms)Region opened successfully at 1731057750195 (+2 ms) 2024-11-08T09:22:30,202 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731057750102 2024-11-08T09:22:30,214 DEBUG [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T09:22:30,214 INFO [RS_OPEN_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T09:22:30,216 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:22:30,218 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cf4d76213909,39151,1731057746690, state=OPEN 2024-11-08T09:22:30,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T09:22:30,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T09:22:30,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T09:22:30,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T09:22:30,345 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:30,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T09:22:30,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cf4d76213909,39151,1731057746690 in 416 msec 2024-11-08T09:22:30,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T09:22:30,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 969 msec 2024-11-08T09:22:30,362 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T09:22:30,362 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T09:22:30,380 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:30,381 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:30,398 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:30,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48179, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:30,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2320 sec 2024-11-08T09:22:30,421 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731057750421, completionTime=-1 2024-11-08T09:22:30,424 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T09:22:30,424 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T09:22:30,446 INFO [master/cf4d76213909:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-08T09:22:30,446 INFO [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731057810446 2024-11-08T09:22:30,447 INFO [master/cf4d76213909:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731057870447 2024-11-08T09:22:30,447 INFO [master/cf4d76213909:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 22 msec 2024-11-08T09:22:30,449 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,35807,1731057745600-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:30,449 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,35807,1731057745600-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:30,449 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,35807,1731057745600-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:30,450 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cf4d76213909:35807, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:30,451 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:30,451 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:30,456 DEBUG [master/cf4d76213909:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T09:22:30,478 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.539sec 2024-11-08T09:22:30,479 INFO [master/cf4d76213909:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T09:22:30,481 INFO [master/cf4d76213909:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T09:22:30,482 INFO [master/cf4d76213909:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T09:22:30,483 INFO [master/cf4d76213909:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T09:22:30,483 INFO [master/cf4d76213909:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T09:22:30,484 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,35807,1731057745600-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T09:22:30,485 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,35807,1731057745600-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T09:22:30,492 DEBUG [master/cf4d76213909:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T09:22:30,493 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T09:22:30,494 INFO [master/cf4d76213909:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cf4d76213909,35807,1731057745600-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T09:22:30,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c90ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:30,533 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T09:22:30,533 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T09:22:30,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:30,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:30,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:30,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:30,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:30,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba7d24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:30,558 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:30,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:30,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cf4d76213909,35807,1731057745600 2024-11-08T09:22:30,598 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=578, ProcessCount=11, AvailableMemoryMB=6587 2024-11-08T09:22:30,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T09:22:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,35807,1731057745600 2024-11-08T09:22:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-08T09:22:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T09:22:30,635 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T09:22:30,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:22:30,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:22:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:30,658 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T09:22:30,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-08T09:22:30,661 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T09:22:30,687 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T09:22:30,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741835_1011 (size=959) 2024-11-08T09:22:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T09:22:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T09:22:31,104 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:22:31,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741836_1012 (size=53) 2024-11-08T09:22:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T09:22:31,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:22:31,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6f50fcf560ee3a7bd187b06bd37f77bb, disabling compactions & flushes 2024-11-08T09:22:31,515 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:31,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:31,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. after waiting 0 ms 2024-11-08T09:22:31,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:31,515 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:31,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6f50fcf560ee3a7bd187b06bd37f77bb: Waiting for close lock at 1731057751515Disabling compacts and flushes for region at 1731057751515Disabling writes for close at 1731057751515Writing region close event to WAL at 1731057751515Closed at 1731057751515 2024-11-08T09:22:31,518 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T09:22:31,523 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731057751518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731057751518"}]},"ts":"1731057751518"} 2024-11-08T09:22:31,528 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T09:22:31,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T09:22:31,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057751531"}]},"ts":"1731057751531"} 2024-11-08T09:22:31,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T09:22:31,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, ASSIGN}] 2024-11-08T09:22:31,544 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, ASSIGN 2024-11-08T09:22:31,546 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, ASSIGN; state=OFFLINE, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=false 2024-11-08T09:22:31,697 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6f50fcf560ee3a7bd187b06bd37f77bb, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:22:31,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, ASSIGN because future has completed 2024-11-08T09:22:31,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:22:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T09:22:31,868 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:31,868 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:22:31,869 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,869 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:22:31,869 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,869 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,872 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,875 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f50fcf560ee3a7bd187b06bd37f77bb columnFamilyName A 2024-11-08T09:22:31,875 DEBUG [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:31,876 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] regionserver.HStore(327): Store=6f50fcf560ee3a7bd187b06bd37f77bb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:22:31,877 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,880 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f50fcf560ee3a7bd187b06bd37f77bb columnFamilyName B 2024-11-08T09:22:31,880 DEBUG [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:31,881 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] regionserver.HStore(327): Store=6f50fcf560ee3a7bd187b06bd37f77bb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:22:31,882 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,885 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f50fcf560ee3a7bd187b06bd37f77bb columnFamilyName C 2024-11-08T09:22:31,885 DEBUG [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:31,886 INFO [StoreOpener-6f50fcf560ee3a7bd187b06bd37f77bb-1 {}] regionserver.HStore(327): Store=6f50fcf560ee3a7bd187b06bd37f77bb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:22:31,886 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,889 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,890 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,892 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,892 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,894 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:22:31,896 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,901 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:22:31,902 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6f50fcf560ee3a7bd187b06bd37f77bb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64003440, jitterRate=-0.046274423599243164}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:22:31,902 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:31,904 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6f50fcf560ee3a7bd187b06bd37f77bb: Running coprocessor pre-open hook at 1731057751870Writing region info on filesystem at 1731057751870Initializing all the Stores at 1731057751871 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057751872 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057751872Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057751872Cleaning up temporary data from old regions at 1731057751893 (+21 ms)Running coprocessor post-open hooks at 1731057751902 (+9 ms)Region opened successfully at 1731057751904 (+2 ms) 2024-11-08T09:22:31,906 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., pid=6, masterSystemTime=1731057751859 2024-11-08T09:22:31,910 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:31,910 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:31,911 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6f50fcf560ee3a7bd187b06bd37f77bb, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:22:31,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:22:31,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T09:22:31,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 in 215 msec 2024-11-08T09:22:31,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T09:22:31,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, ASSIGN in 382 msec 2024-11-08T09:22:31,932 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T09:22:31,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057751932"}]},"ts":"1731057751932"} 2024-11-08T09:22:31,936 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T09:22:31,937 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T09:22:31,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2910 sec 2024-11-08T09:22:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T09:22:32,813 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:32,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8b07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a04f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435bed7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,827 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T09:22:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,35807,1731057745600 2024-11-08T09:22:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bf45604 2024-11-08T09:22:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T09:22:32,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T09:22:32,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff18d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,854 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109bd06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,861 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af20e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,870 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,877 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7387a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,882 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a031a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,890 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,893 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276a94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,900 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267419b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d403639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,904 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9550bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,907 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b38edab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d66d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,911 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020fb6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,914 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57865cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be0d766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:22:32,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:22:32,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:22:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:22:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:22:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463070d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:22:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:22:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:32,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:22:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5c6f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:22:32,926 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,926 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,928 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,930 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,931 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,933 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,942 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:32,951 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,951 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-08T09:22:32,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,958 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,963 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T09:22:32,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:32,977 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:22:32,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:22:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:22:32,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:22:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:22:33,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:33,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:22:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T09:22:33,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T09:22:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/0faa37d5a979434a908cb43edd396fc7 is 50, key is test_row_0/A:col10/1731057752983/Put/seqid=0 2024-11-08T09:22:33,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057813220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057813240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057813240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057813245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057813251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741837_1013 (size=12001) 2024-11-08T09:22:33,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/0faa37d5a979434a908cb43edd396fc7 2024-11-08T09:22:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T09:22:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T09:22:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a1b991f93d374902b6afc53e611f52f5 is 50, key is test_row_0/B:col10/1731057752983/Put/seqid=0 2024-11-08T09:22:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057813395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057813393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057813395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057813395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057813396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741838_1014 (size=12001) 2024-11-08T09:22:33,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a1b991f93d374902b6afc53e611f52f5 2024-11-08T09:22:33,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/77a30783f6e341c6b8d1d89fa0287bab is 50, key is test_row_0/C:col10/1731057752983/Put/seqid=0 2024-11-08T09:22:33,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741839_1015 (size=12001) 2024-11-08T09:22:33,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T09:22:33,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:33,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T09:22:33,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057813602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057813613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057813613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057813613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057813613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T09:22:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T09:22:33,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:33,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/77a30783f6e341c6b8d1d89fa0287bab 2024-11-08T09:22:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/0faa37d5a979434a908cb43edd396fc7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/0faa37d5a979434a908cb43edd396fc7 2024-11-08T09:22:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057813912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057813923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057813932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057813932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057813933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:33,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/0faa37d5a979434a908cb43edd396fc7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T09:22:33,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a1b991f93d374902b6afc53e611f52f5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a1b991f93d374902b6afc53e611f52f5 2024-11-08T09:22:33,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T09:22:33,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a1b991f93d374902b6afc53e611f52f5, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T09:22:33,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:33,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:33,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:33,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/77a30783f6e341c6b8d1d89fa0287bab as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/77a30783f6e341c6b8d1d89fa0287bab 2024-11-08T09:22:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:34,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/77a30783f6e341c6b8d1d89fa0287bab, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T09:22:34,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6f50fcf560ee3a7bd187b06bd37f77bb in 971ms, sequenceid=13, compaction requested=false 2024-11-08T09:22:34,019 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T09:22:34,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T09:22:34,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T09:22:34,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:34,146 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:22:34,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/de692b97388f46c182de0cf28fcd98e2 is 50, key is test_row_0/A:col10/1731057753205/Put/seqid=0 2024-11-08T09:22:34,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741840_1016 (size=12001) 2024-11-08T09:22:34,195 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/de692b97388f46c182de0cf28fcd98e2 2024-11-08T09:22:34,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/607ffacc1d704c75813a30376a6f268e is 50, key is test_row_0/B:col10/1731057753205/Put/seqid=0 2024-11-08T09:22:34,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741841_1017 (size=12001) 2024-11-08T09:22:34,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057814469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057814469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057814473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057814479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057814481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057814594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057814594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057814593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057814604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057814603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,650 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/607ffacc1d704c75813a30376a6f268e 2024-11-08T09:22:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/93aa794b1ecf44c7a4d1409f2728cd4c is 50, key is test_row_0/C:col10/1731057753205/Put/seqid=0 2024-11-08T09:22:34,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741842_1018 (size=12001) 2024-11-08T09:22:34,719 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/93aa794b1ecf44c7a4d1409f2728cd4c 2024-11-08T09:22:34,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/de692b97388f46c182de0cf28fcd98e2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/de692b97388f46c182de0cf28fcd98e2 2024-11-08T09:22:34,758 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/de692b97388f46c182de0cf28fcd98e2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T09:22:34,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/607ffacc1d704c75813a30376a6f268e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/607ffacc1d704c75813a30376a6f268e 2024-11-08T09:22:34,777 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/607ffacc1d704c75813a30376a6f268e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T09:22:34,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/93aa794b1ecf44c7a4d1409f2728cd4c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/93aa794b1ecf44c7a4d1409f2728cd4c 2024-11-08T09:22:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057814802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057814803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,817 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/93aa794b1ecf44c7a4d1409f2728cd4c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T09:22:34,819 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6f50fcf560ee3a7bd187b06bd37f77bb in 673ms, sequenceid=37, compaction requested=false 2024-11-08T09:22:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-08T09:22:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-08T09:22:34,833 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:34,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T09:22:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:34,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T09:22:34,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8750 sec 2024-11-08T09:22:34,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9080 sec 2024-11-08T09:22:34,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/567328cd584e455480f869ecc02cae16 is 50, key is test_row_0/A:col10/1731057754462/Put/seqid=0 2024-11-08T09:22:34,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741843_1019 (size=12001) 2024-11-08T09:22:34,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057814945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057814954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:34,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057814961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057815062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057815062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057815072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T09:22:35,112 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:35,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-08T09:22:35,117 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:35,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:35,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T09:22:35,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057815122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057815122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T09:22:35,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057815272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057815272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/567328cd584e455480f869ecc02cae16 2024-11-08T09:22:35,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057815282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/2ce2a7fcd01b41d696d646157d55b288 is 50, key is test_row_0/B:col10/1731057754462/Put/seqid=0 2024-11-08T09:22:35,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741844_1020 (size=12001) 2024-11-08T09:22:35,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:35,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T09:22:35,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057815582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057815582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057815602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057815642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057815642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:35,693 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T09:22:35,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:35,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T09:22:35,787 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T09:22:35,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/2ce2a7fcd01b41d696d646157d55b288 2024-11-08T09:22:35,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/7224f5a2936547f9b851dee225d05415 is 50, key is test_row_0/C:col10/1731057754462/Put/seqid=0 2024-11-08T09:22:35,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741845_1021 (size=12001) 2024-11-08T09:22:35,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:35,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:36,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:36,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:36,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:36,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T09:22:36,081 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T09:22:36,083 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T09:22:36,083 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T09:22:36,083 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T09:22:36,083 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T09:22:36,085 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T09:22:36,085 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T09:22:36,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057816102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057816102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057816122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:36,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:36,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:36,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/7224f5a2936547f9b851dee225d05415 2024-11-08T09:22:36,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/567328cd584e455480f869ecc02cae16 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/567328cd584e455480f869ecc02cae16 2024-11-08T09:22:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T09:22:36,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/567328cd584e455480f869ecc02cae16, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T09:22:36,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/2ce2a7fcd01b41d696d646157d55b288 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/2ce2a7fcd01b41d696d646157d55b288 2024-11-08T09:22:36,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/2ce2a7fcd01b41d696d646157d55b288, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T09:22:36,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/7224f5a2936547f9b851dee225d05415 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/7224f5a2936547f9b851dee225d05415 2024-11-08T09:22:36,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/7224f5a2936547f9b851dee225d05415, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T09:22:36,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1486ms, sequenceid=51, compaction requested=true 2024-11-08T09:22:36,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:36,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:36,329 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:36,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:36,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:36,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:36,331 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:36,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:36,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:36,337 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:36,340 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:36,340 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,341 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a1b991f93d374902b6afc53e611f52f5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/607ffacc1d704c75813a30376a6f268e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/2ce2a7fcd01b41d696d646157d55b288] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=35.2 K 2024-11-08T09:22:36,342 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:36,342 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:36,342 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,343 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/0faa37d5a979434a908cb43edd396fc7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/de692b97388f46c182de0cf28fcd98e2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/567328cd584e455480f869ecc02cae16] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=35.2 K 2024-11-08T09:22:36,347 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a1b991f93d374902b6afc53e611f52f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731057752983 2024-11-08T09:22:36,348 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0faa37d5a979434a908cb43edd396fc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731057752983 2024-11-08T09:22:36,348 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 607ffacc1d704c75813a30376a6f268e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731057753205 2024-11-08T09:22:36,350 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce2a7fcd01b41d696d646157d55b288, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731057754462 2024-11-08T09:22:36,350 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting de692b97388f46c182de0cf28fcd98e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731057753205 2024-11-08T09:22:36,351 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567328cd584e455480f869ecc02cae16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731057754462 2024-11-08T09:22:36,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T09:22:36,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,368 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T09:22:36,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/5f46710c08a944a28ed49dfcacec0e0a is 50, key is test_row_0/A:col10/1731057754948/Put/seqid=0 2024-11-08T09:22:36,400 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#11 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:36,401 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/153c276385c8439686a59d44c0d221fb is 50, key is test_row_0/B:col10/1731057754462/Put/seqid=0 2024-11-08T09:22:36,410 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#9 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:36,411 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/3f34bf8fb27a4da7975011bc48c82ff1 is 50, key is test_row_0/A:col10/1731057754462/Put/seqid=0 2024-11-08T09:22:36,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741846_1022 (size=12001) 2024-11-08T09:22:36,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741847_1023 (size=12104) 2024-11-08T09:22:36,442 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/153c276385c8439686a59d44c0d221fb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/153c276385c8439686a59d44c0d221fb 2024-11-08T09:22:36,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741848_1024 (size=12104) 2024-11-08T09:22:36,468 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into 153c276385c8439686a59d44c0d221fb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:36,468 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:36,471 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=13, startTime=1731057756331; duration=0sec 2024-11-08T09:22:36,472 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:36,472 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:36,472 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:36,478 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:36,478 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:36,478 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,478 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/77a30783f6e341c6b8d1d89fa0287bab, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/93aa794b1ecf44c7a4d1409f2728cd4c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/7224f5a2936547f9b851dee225d05415] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=35.2 K 2024-11-08T09:22:36,480 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a30783f6e341c6b8d1d89fa0287bab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731057752983 2024-11-08T09:22:36,481 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 93aa794b1ecf44c7a4d1409f2728cd4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731057753205 2024-11-08T09:22:36,483 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 7224f5a2936547f9b851dee225d05415, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731057754462 2024-11-08T09:22:36,518 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:36,519 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/9405b6dc0cd94b24803fb4c877358020 is 50, key is test_row_0/C:col10/1731057754462/Put/seqid=0 2024-11-08T09:22:36,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741849_1025 (size=12104) 2024-11-08T09:22:36,564 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/9405b6dc0cd94b24803fb4c877358020 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/9405b6dc0cd94b24803fb4c877358020 2024-11-08T09:22:36,577 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into 9405b6dc0cd94b24803fb4c877358020(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:36,577 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:36,577 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=13, startTime=1731057756331; duration=0sec 2024-11-08T09:22:36,577 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:36,577 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:36,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057816695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:36,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057816701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057816812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:36,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057816812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:36,821 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/5f46710c08a944a28ed49dfcacec0e0a 2024-11-08T09:22:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/882d3ed4dd124538ae861a304f12c85c is 50, key is test_row_0/B:col10/1731057754948/Put/seqid=0 2024-11-08T09:22:36,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741850_1026 (size=12001) 2024-11-08T09:22:36,860 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/882d3ed4dd124538ae861a304f12c85c 2024-11-08T09:22:36,879 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/3f34bf8fb27a4da7975011bc48c82ff1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/3f34bf8fb27a4da7975011bc48c82ff1 2024-11-08T09:22:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/2066359bebac43e89bec3c183bbbfa83 is 50, key is test_row_0/C:col10/1731057754948/Put/seqid=0 2024-11-08T09:22:36,899 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into 3f34bf8fb27a4da7975011bc48c82ff1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:36,900 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:36,900 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=13, startTime=1731057756322; duration=0sec 2024-11-08T09:22:36,900 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:36,900 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:36,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741851_1027 (size=12001) 2024-11-08T09:22:36,909 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/2066359bebac43e89bec3c183bbbfa83 2024-11-08T09:22:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/5f46710c08a944a28ed49dfcacec0e0a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/5f46710c08a944a28ed49dfcacec0e0a 2024-11-08T09:22:36,940 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/5f46710c08a944a28ed49dfcacec0e0a, entries=150, sequenceid=73, filesize=11.7 K 2024-11-08T09:22:36,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/882d3ed4dd124538ae861a304f12c85c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/882d3ed4dd124538ae861a304f12c85c 2024-11-08T09:22:36,959 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/882d3ed4dd124538ae861a304f12c85c, entries=150, sequenceid=73, filesize=11.7 K 2024-11-08T09:22:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/2066359bebac43e89bec3c183bbbfa83 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2066359bebac43e89bec3c183bbbfa83 2024-11-08T09:22:36,980 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2066359bebac43e89bec3c183bbbfa83, entries=150, sequenceid=73, filesize=11.7 K 2024-11-08T09:22:36,982 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6f50fcf560ee3a7bd187b06bd37f77bb in 613ms, sequenceid=73, compaction requested=false 2024-11-08T09:22:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-08T09:22:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-08T09:22:36,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T09:22:36,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-08T09:22:36,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8790 sec 2024-11-08T09:22:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:37,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:37,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T09:22:37,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/39f73fb4bceb475e9d56c8dd634cfece is 50, key is test_row_0/A:col10/1731057757023/Put/seqid=0 2024-11-08T09:22:37,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741852_1028 (size=14341) 2024-11-08T09:22:37,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/39f73fb4bceb475e9d56c8dd634cfece 2024-11-08T09:22:37,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/c4ab6bda3c9d417ca380bc583602ded5 is 50, key is test_row_0/B:col10/1731057757023/Put/seqid=0 2024-11-08T09:22:37,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741853_1029 (size=12001) 2024-11-08T09:22:37,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/c4ab6bda3c9d417ca380bc583602ded5 2024-11-08T09:22:37,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057817146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057817144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057817147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057817155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057817156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/eeae8acd337d46caa4519ef3355e4fc3 is 50, key is test_row_0/C:col10/1731057757023/Put/seqid=0 2024-11-08T09:22:37,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741854_1030 (size=12001) 2024-11-08T09:22:37,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/eeae8acd337d46caa4519ef3355e4fc3 2024-11-08T09:22:37,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/39f73fb4bceb475e9d56c8dd634cfece as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/39f73fb4bceb475e9d56c8dd634cfece 2024-11-08T09:22:37,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/39f73fb4bceb475e9d56c8dd634cfece, entries=200, sequenceid=91, filesize=14.0 K 2024-11-08T09:22:37,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/c4ab6bda3c9d417ca380bc583602ded5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/c4ab6bda3c9d417ca380bc583602ded5 2024-11-08T09:22:37,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/c4ab6bda3c9d417ca380bc583602ded5, entries=150, sequenceid=91, filesize=11.7 K 2024-11-08T09:22:37,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/eeae8acd337d46caa4519ef3355e4fc3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eeae8acd337d46caa4519ef3355e4fc3 2024-11-08T09:22:37,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eeae8acd337d46caa4519ef3355e4fc3, entries=150, sequenceid=91, filesize=11.7 K 2024-11-08T09:22:37,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for 6f50fcf560ee3a7bd187b06bd37f77bb in 236ms, sequenceid=91, compaction requested=true 2024-11-08T09:22:37,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:37,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:37,262 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:37,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:37,262 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:37,264 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:37,265 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:37,265 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,265 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/3f34bf8fb27a4da7975011bc48c82ff1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/5f46710c08a944a28ed49dfcacec0e0a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/39f73fb4bceb475e9d56c8dd634cfece] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=37.5 K 2024-11-08T09:22:37,265 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:37,265 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:37,266 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,266 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/153c276385c8439686a59d44c0d221fb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/882d3ed4dd124538ae861a304f12c85c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/c4ab6bda3c9d417ca380bc583602ded5] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=35.3 K 2024-11-08T09:22:37,266 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f34bf8fb27a4da7975011bc48c82ff1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731057754462 2024-11-08T09:22:37,267 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 153c276385c8439686a59d44c0d221fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731057754462 2024-11-08T09:22:37,267 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f46710c08a944a28ed49dfcacec0e0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731057754942 2024-11-08T09:22:37,267 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 882d3ed4dd124538ae861a304f12c85c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731057754942 2024-11-08T09:22:37,268 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39f73fb4bceb475e9d56c8dd634cfece, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731057756674 2024-11-08T09:22:37,269 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting c4ab6bda3c9d417ca380bc583602ded5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731057756674 2024-11-08T09:22:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T09:22:37,272 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:37,273 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-08T09:22:37,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:37,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:22:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T09:22:37,284 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:37,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:37,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/36df0bc47a26424aa09c1576435c5bb5 is 50, key is test_row_0/A:col10/1731057757140/Put/seqid=0 2024-11-08T09:22:37,303 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:37,304 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/61074a6545be46439b919fa4bd4f3775 is 50, key is test_row_0/B:col10/1731057757023/Put/seqid=0 2024-11-08T09:22:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057817305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057817309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,315 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:37,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,316 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/7431cfae884b4b4992651b98f754ced4 is 50, key is test_row_0/A:col10/1731057757023/Put/seqid=0 2024-11-08T09:22:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057817311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057817314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057817315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741856_1032 (size=12207) 2024-11-08T09:22:37,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741855_1031 (size=14341) 2024-11-08T09:22:37,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/36df0bc47a26424aa09c1576435c5bb5 2024-11-08T09:22:37,357 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/61074a6545be46439b919fa4bd4f3775 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/61074a6545be46439b919fa4bd4f3775 2024-11-08T09:22:37,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741857_1033 (size=12207) 2024-11-08T09:22:37,378 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/7431cfae884b4b4992651b98f754ced4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7431cfae884b4b4992651b98f754ced4 2024-11-08T09:22:37,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/389517c9303048d787abbe2db386fdd2 is 50, key is test_row_0/B:col10/1731057757140/Put/seqid=0 2024-11-08T09:22:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T09:22:37,392 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into 7431cfae884b4b4992651b98f754ced4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:37,396 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:37,396 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=13, startTime=1731057757262; duration=0sec 2024-11-08T09:22:37,396 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:37,396 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:37,396 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:37,398 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into 61074a6545be46439b919fa4bd4f3775(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:37,398 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:37,398 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=13, startTime=1731057757262; duration=0sec 2024-11-08T09:22:37,399 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:37,399 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:37,400 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:37,400 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:37,400 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,401 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/9405b6dc0cd94b24803fb4c877358020, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2066359bebac43e89bec3c183bbbfa83, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eeae8acd337d46caa4519ef3355e4fc3] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=35.3 K 2024-11-08T09:22:37,402 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9405b6dc0cd94b24803fb4c877358020, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731057754462 2024-11-08T09:22:37,403 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2066359bebac43e89bec3c183bbbfa83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731057754942 2024-11-08T09:22:37,405 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeae8acd337d46caa4519ef3355e4fc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731057756674 2024-11-08T09:22:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057817422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057817422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057817431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057817432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057817432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741858_1034 (size=12001) 2024-11-08T09:22:37,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T09:22:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:37,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,445 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:37,446 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/3614a1381946470f86010a961f8a6489 is 50, key is test_row_0/C:col10/1731057757023/Put/seqid=0 2024-11-08T09:22:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741859_1035 (size=12207) 2024-11-08T09:22:37,469 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/3614a1381946470f86010a961f8a6489 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3614a1381946470f86010a961f8a6489 2024-11-08T09:22:37,483 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into 3614a1381946470f86010a961f8a6489(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:37,483 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:37,483 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=13, startTime=1731057757263; duration=0sec 2024-11-08T09:22:37,483 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:37,483 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:37,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T09:22:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T09:22:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057817632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057817641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057817642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057817642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057817642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T09:22:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/389517c9303048d787abbe2db386fdd2 2024-11-08T09:22:37,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/348668329095479f9ede52de24f03c8e is 50, key is test_row_0/C:col10/1731057757140/Put/seqid=0 2024-11-08T09:22:37,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741860_1036 (size=12001) 2024-11-08T09:22:37,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/348668329095479f9ede52de24f03c8e 2024-11-08T09:22:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/36df0bc47a26424aa09c1576435c5bb5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/36df0bc47a26424aa09c1576435c5bb5 2024-11-08T09:22:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T09:22:37,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T09:22:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:37,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:37,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/36df0bc47a26424aa09c1576435c5bb5, entries=200, sequenceid=115, filesize=14.0 K 2024-11-08T09:22:37,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/389517c9303048d787abbe2db386fdd2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/389517c9303048d787abbe2db386fdd2 2024-11-08T09:22:37,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057817942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/389517c9303048d787abbe2db386fdd2, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T09:22:37,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/348668329095479f9ede52de24f03c8e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/348668329095479f9ede52de24f03c8e 2024-11-08T09:22:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057817951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057817951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057817952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057817952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:37,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/348668329095479f9ede52de24f03c8e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T09:22:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:37,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 6f50fcf560ee3a7bd187b06bd37f77bb in 688ms, sequenceid=115, compaction requested=false 2024-11-08T09:22:37,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:38,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T09:22:38,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:38,070 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:22:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/b3db1c0e00a24159b439a1d6ca8f983f is 50, key is test_row_0/A:col10/1731057757308/Put/seqid=0 2024-11-08T09:22:38,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741861_1037 (size=12001) 2024-11-08T09:22:38,102 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/b3db1c0e00a24159b439a1d6ca8f983f 2024-11-08T09:22:38,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/ac3d698795564012a0a2253a6b522339 is 50, key is test_row_0/B:col10/1731057757308/Put/seqid=0 2024-11-08T09:22:38,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741862_1038 (size=12001) 2024-11-08T09:22:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T09:22:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:38,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057818526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057818526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057818527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057818529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057818534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,543 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/ac3d698795564012a0a2253a6b522339 2024-11-08T09:22:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/21d6b9cb1ef54d768d8b72fb863d4e84 is 50, key is test_row_0/C:col10/1731057757308/Put/seqid=0 2024-11-08T09:22:38,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741863_1039 (size=12001) 2024-11-08T09:22:38,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057818642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057818643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057818643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057818643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057818644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057818851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057818852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057818852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057818852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057818852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:38,974 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/21d6b9cb1ef54d768d8b72fb863d4e84 2024-11-08T09:22:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/b3db1c0e00a24159b439a1d6ca8f983f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b3db1c0e00a24159b439a1d6ca8f983f 2024-11-08T09:22:38,996 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b3db1c0e00a24159b439a1d6ca8f983f, entries=150, sequenceid=131, filesize=11.7 K 2024-11-08T09:22:38,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/ac3d698795564012a0a2253a6b522339 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac3d698795564012a0a2253a6b522339 2024-11-08T09:22:39,009 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac3d698795564012a0a2253a6b522339, entries=150, sequenceid=131, filesize=11.7 K 2024-11-08T09:22:39,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/21d6b9cb1ef54d768d8b72fb863d4e84 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/21d6b9cb1ef54d768d8b72fb863d4e84 2024-11-08T09:22:39,023 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/21d6b9cb1ef54d768d8b72fb863d4e84, entries=150, sequenceid=131, filesize=11.7 K 2024-11-08T09:22:39,025 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6f50fcf560ee3a7bd187b06bd37f77bb in 954ms, sequenceid=131, compaction requested=true 2024-11-08T09:22:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-08T09:22:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-08T09:22:39,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-08T09:22:39,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7430 sec 2024-11-08T09:22:39,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7590 sec 2024-11-08T09:22:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:39,167 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:39,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:22:39,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/8f0288bb622244b6b53d28b2294996bf is 50, key is test_row_0/A:col10/1731057758520/Put/seqid=0 2024-11-08T09:22:39,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057819174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057819177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057819177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057819177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057819180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741864_1040 (size=14541) 2024-11-08T09:22:39,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/8f0288bb622244b6b53d28b2294996bf 2024-11-08T09:22:39,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/561fc9ac67644c58b9534ff8e6c36d9a is 50, key is test_row_0/B:col10/1731057758520/Put/seqid=0 2024-11-08T09:22:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741865_1041 (size=12151) 2024-11-08T09:22:39,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/561fc9ac67644c58b9534ff8e6c36d9a 2024-11-08T09:22:39,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/2457e865544e4a5cb1be519d386c5d7e is 50, key is test_row_0/C:col10/1731057758520/Put/seqid=0 2024-11-08T09:22:39,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741866_1042 (size=12151) 2024-11-08T09:22:39,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057819292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057819292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057819292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057819292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057819293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T09:22:39,432 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:39,433 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-08T09:22:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T09:22:39,435 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:39,436 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:39,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057819501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057819502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057819502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057819502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057819502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T09:22:39,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/2457e865544e4a5cb1be519d386c5d7e 2024-11-08T09:22:39,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/8f0288bb622244b6b53d28b2294996bf as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8f0288bb622244b6b53d28b2294996bf 2024-11-08T09:22:39,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8f0288bb622244b6b53d28b2294996bf, entries=200, sequenceid=158, filesize=14.2 K 2024-11-08T09:22:39,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/561fc9ac67644c58b9534ff8e6c36d9a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/561fc9ac67644c58b9534ff8e6c36d9a 2024-11-08T09:22:39,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/561fc9ac67644c58b9534ff8e6c36d9a, entries=150, sequenceid=158, filesize=11.9 K 2024-11-08T09:22:39,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/2457e865544e4a5cb1be519d386c5d7e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2457e865544e4a5cb1be519d386c5d7e 2024-11-08T09:22:39,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2457e865544e4a5cb1be519d386c5d7e, entries=150, sequenceid=158, filesize=11.9 K 2024-11-08T09:22:39,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:39,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:39,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 6f50fcf560ee3a7bd187b06bd37f77bb in 578ms, sequenceid=158, compaction requested=true 2024-11-08T09:22:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:39,746 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:22:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:39,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T09:22:39,748 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:22:39,748 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:39,748 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:22:39,748 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,749 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7431cfae884b4b4992651b98f754ced4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/36df0bc47a26424aa09c1576435c5bb5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b3db1c0e00a24159b439a1d6ca8f983f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8f0288bb622244b6b53d28b2294996bf] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=51.8 K 2024-11-08T09:22:39,750 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7431cfae884b4b4992651b98f754ced4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731057756674 2024-11-08T09:22:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,750 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36df0bc47a26424aa09c1576435c5bb5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731057757140 2024-11-08T09:22:39,751 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3db1c0e00a24159b439a1d6ca8f983f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731057757307 2024-11-08T09:22:39,751 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:22:39,751 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:39,751 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,752 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/61074a6545be46439b919fa4bd4f3775, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/389517c9303048d787abbe2db386fdd2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac3d698795564012a0a2253a6b522339, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/561fc9ac67644c58b9534ff8e6c36d9a] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=47.2 K 2024-11-08T09:22:39,752 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0288bb622244b6b53d28b2294996bf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731057758520 2024-11-08T09:22:39,753 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 61074a6545be46439b919fa4bd4f3775, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731057756674 2024-11-08T09:22:39,755 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 389517c9303048d787abbe2db386fdd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731057757140 2024-11-08T09:22:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T09:22:39,756 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3d698795564012a0a2253a6b522339, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731057757307 2024-11-08T09:22:39,757 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 561fc9ac67644c58b9534ff8e6c36d9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731057758520 2024-11-08T09:22:39,778 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:39,779 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/eae161419faf4a6a9e7f0752ea150977 is 50, key is test_row_0/A:col10/1731057758520/Put/seqid=0 2024-11-08T09:22:39,792 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:39,792 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/4246eb446abf41e8964cc029cf6d8fd7 is 50, key is test_row_0/B:col10/1731057758520/Put/seqid=0 2024-11-08T09:22:39,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741867_1043 (size=12493) 2024-11-08T09:22:39,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741868_1044 (size=12493) 2024-11-08T09:22:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:39,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:22:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:39,826 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/4246eb446abf41e8964cc029cf6d8fd7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4246eb446abf41e8964cc029cf6d8fd7 2024-11-08T09:22:39,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/7c8486678c294cc9a583854f280e0f6d is 50, key is test_row_0/A:col10/1731057759817/Put/seqid=0 2024-11-08T09:22:39,843 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into 4246eb446abf41e8964cc029cf6d8fd7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:39,843 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:39,843 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=12, startTime=1731057759746; duration=0sec 2024-11-08T09:22:39,843 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:39,843 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:39,843 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:22:39,845 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:22:39,845 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:39,846 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,846 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3614a1381946470f86010a961f8a6489, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/348668329095479f9ede52de24f03c8e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/21d6b9cb1ef54d768d8b72fb863d4e84, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2457e865544e4a5cb1be519d386c5d7e] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=47.2 K 2024-11-08T09:22:39,848 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 3614a1381946470f86010a961f8a6489, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731057756674 2024-11-08T09:22:39,850 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 348668329095479f9ede52de24f03c8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731057757140 2024-11-08T09:22:39,850 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d6b9cb1ef54d768d8b72fb863d4e84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731057757307 2024-11-08T09:22:39,851 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2457e865544e4a5cb1be519d386c5d7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731057758520 2024-11-08T09:22:39,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741869_1045 (size=12151) 2024-11-08T09:22:39,870 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:39,871 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/69a5e35c16bd45dcbde1522409660025 is 50, key is test_row_0/C:col10/1731057758520/Put/seqid=0 2024-11-08T09:22:39,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057819885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057819887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741870_1046 (size=12493) 2024-11-08T09:22:39,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:39,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:39,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057819895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057819897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057819897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:39,939 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/69a5e35c16bd45dcbde1522409660025 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/69a5e35c16bd45dcbde1522409660025 2024-11-08T09:22:39,958 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into 69a5e35c16bd45dcbde1522409660025(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:39,958 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:39,958 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=12, startTime=1731057759746; duration=0sec 2024-11-08T09:22:39,959 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:39,959 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:40,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057820002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057820002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057820021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057820022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057820022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:40,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T09:22:40,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057820212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,215 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/eae161419faf4a6a9e7f0752ea150977 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/eae161419faf4a6a9e7f0752ea150977 2024-11-08T09:22:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,225 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into eae161419faf4a6a9e7f0752ea150977(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:40,225 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:40,225 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=12, startTime=1731057759745; duration=0sec 2024-11-08T09:22:40,225 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:40,225 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:40,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057820221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057820232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057820232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057820232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/7c8486678c294cc9a583854f280e0f6d 2024-11-08T09:22:40,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/ac59503d178342e99e01c6d6ddfa5b5e is 50, key is test_row_0/B:col10/1731057759817/Put/seqid=0 2024-11-08T09:22:40,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741871_1047 (size=12151) 2024-11-08T09:22:40,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057820521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057820531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057820542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057820542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057820542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T09:22:40,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:40,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/ac59503d178342e99e01c6d6ddfa5b5e 2024-11-08T09:22:40,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/55916363ac994d42ae9d2d9cc9c8141b is 50, key is test_row_0/C:col10/1731057759817/Put/seqid=0 2024-11-08T09:22:40,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741872_1048 (size=12151) 2024-11-08T09:22:40,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/55916363ac994d42ae9d2d9cc9c8141b 2024-11-08T09:22:40,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/7c8486678c294cc9a583854f280e0f6d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7c8486678c294cc9a583854f280e0f6d 2024-11-08T09:22:40,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7c8486678c294cc9a583854f280e0f6d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T09:22:40,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/ac59503d178342e99e01c6d6ddfa5b5e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac59503d178342e99e01c6d6ddfa5b5e 2024-11-08T09:22:40,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac59503d178342e99e01c6d6ddfa5b5e, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T09:22:40,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/55916363ac994d42ae9d2d9cc9c8141b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/55916363ac994d42ae9d2d9cc9c8141b 2024-11-08T09:22:40,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/55916363ac994d42ae9d2d9cc9c8141b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T09:22:40,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 6f50fcf560ee3a7bd187b06bd37f77bb in 953ms, sequenceid=171, compaction requested=false 2024-11-08T09:22:40,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:40,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T09:22:40,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:40,833 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:22:40,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/ba75d0df22214e249a65308bee5f4fbb is 50, key is test_row_0/A:col10/1731057759890/Put/seqid=0 2024-11-08T09:22:40,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741873_1049 (size=12151) 2024-11-08T09:22:40,848 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/ba75d0df22214e249a65308bee5f4fbb 2024-11-08T09:22:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/7ebd68d304184dd8879086c5a5152ec2 is 50, key is test_row_0/B:col10/1731057759890/Put/seqid=0 2024-11-08T09:22:40,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741874_1050 (size=12151) 2024-11-08T09:22:41,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:41,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057821079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057821080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057821081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057821082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057821082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057821192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057821192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057821192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057821192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057821192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,280 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/7ebd68d304184dd8879086c5a5152ec2 2024-11-08T09:22:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/56ae784dd5d94ec59919d1cd8e230b4d is 50, key is test_row_0/C:col10/1731057759890/Put/seqid=0 2024-11-08T09:22:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741875_1051 (size=12151) 2024-11-08T09:22:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057821401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057821401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057821402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057821402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057821402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T09:22:41,705 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/56ae784dd5d94ec59919d1cd8e230b4d 2024-11-08T09:22:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057821714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057821714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057821714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057821714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057821714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:41,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:41,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/ba75d0df22214e249a65308bee5f4fbb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/ba75d0df22214e249a65308bee5f4fbb 2024-11-08T09:22:41,730 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/ba75d0df22214e249a65308bee5f4fbb, entries=150, sequenceid=199, filesize=11.9 K 2024-11-08T09:22:41,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/7ebd68d304184dd8879086c5a5152ec2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/7ebd68d304184dd8879086c5a5152ec2 2024-11-08T09:22:41,748 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/7ebd68d304184dd8879086c5a5152ec2, entries=150, sequenceid=199, filesize=11.9 K 2024-11-08T09:22:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/56ae784dd5d94ec59919d1cd8e230b4d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/56ae784dd5d94ec59919d1cd8e230b4d 2024-11-08T09:22:41,765 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/56ae784dd5d94ec59919d1cd8e230b4d, entries=150, sequenceid=199, filesize=11.9 K 2024-11-08T09:22:41,767 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 6f50fcf560ee3a7bd187b06bd37f77bb in 934ms, sequenceid=199, compaction requested=true 2024-11-08T09:22:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-08T09:22:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-08T09:22:41,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T09:22:41,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-11-08T09:22:41,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3410 sec 2024-11-08T09:22:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:42,226 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:42,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T09:22:42,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c07bd21e4f1f4ca1b7c2f081f32a667c is 50, key is test_row_0/A:col10/1731057761078/Put/seqid=0 2024-11-08T09:22:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741876_1052 (size=12151) 2024-11-08T09:22:42,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057822247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057822255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057822259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057822259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057822260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057822362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057822371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057822371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057822372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057822372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057822571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057822582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057822582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057822583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057822583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c07bd21e4f1f4ca1b7c2f081f32a667c 2024-11-08T09:22:42,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/5411d6e072b1456091b0ed7666b1da22 is 50, key is test_row_0/B:col10/1731057761078/Put/seqid=0 2024-11-08T09:22:42,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741877_1053 (size=12151) 2024-11-08T09:22:42,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057822881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057822891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057822892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057822892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057822892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:43,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/5411d6e072b1456091b0ed7666b1da22 2024-11-08T09:22:43,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5a0559add7fe4cb2a84960bc3ca38204 is 50, key is test_row_0/C:col10/1731057761078/Put/seqid=0 2024-11-08T09:22:43,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741878_1054 (size=12151) 2024-11-08T09:22:43,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057823392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:43,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057823401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057823401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057823401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:43,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057823412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:43,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5a0559add7fe4cb2a84960bc3ca38204 2024-11-08T09:22:43,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c07bd21e4f1f4ca1b7c2f081f32a667c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c07bd21e4f1f4ca1b7c2f081f32a667c 2024-11-08T09:22:43,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c07bd21e4f1f4ca1b7c2f081f32a667c, entries=150, sequenceid=215, filesize=11.9 K 2024-11-08T09:22:43,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/5411d6e072b1456091b0ed7666b1da22 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/5411d6e072b1456091b0ed7666b1da22 2024-11-08T09:22:43,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/5411d6e072b1456091b0ed7666b1da22, entries=150, sequenceid=215, filesize=11.9 K 2024-11-08T09:22:43,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5a0559add7fe4cb2a84960bc3ca38204 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a0559add7fe4cb2a84960bc3ca38204 2024-11-08T09:22:43,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a0559add7fe4cb2a84960bc3ca38204, entries=150, sequenceid=215, filesize=11.9 K 2024-11-08T09:22:43,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1307ms, sequenceid=215, compaction requested=true 2024-11-08T09:22:43,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:43,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:43,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:43,534 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:22:43,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:43,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:43,534 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:22:43,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:43,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:43,535 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:22:43,536 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:43,536 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:22:43,536 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:43,536 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:43,536 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:43,536 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4246eb446abf41e8964cc029cf6d8fd7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac59503d178342e99e01c6d6ddfa5b5e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/7ebd68d304184dd8879086c5a5152ec2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/5411d6e072b1456091b0ed7666b1da22] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=47.8 K 2024-11-08T09:22:43,536 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/eae161419faf4a6a9e7f0752ea150977, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7c8486678c294cc9a583854f280e0f6d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/ba75d0df22214e249a65308bee5f4fbb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c07bd21e4f1f4ca1b7c2f081f32a667c] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=47.8 K 2024-11-08T09:22:43,537 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae161419faf4a6a9e7f0752ea150977, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731057758520 2024-11-08T09:22:43,537 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 4246eb446abf41e8964cc029cf6d8fd7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731057758520 2024-11-08T09:22:43,537 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c8486678c294cc9a583854f280e0f6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731057759814 2024-11-08T09:22:43,537 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting ac59503d178342e99e01c6d6ddfa5b5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731057759814 2024-11-08T09:22:43,538 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba75d0df22214e249a65308bee5f4fbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057759883 2024-11-08T09:22:43,538 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ebd68d304184dd8879086c5a5152ec2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057759883 2024-11-08T09:22:43,539 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c07bd21e4f1f4ca1b7c2f081f32a667c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731057761078 2024-11-08T09:22:43,539 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5411d6e072b1456091b0ed7666b1da22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731057761078 2024-11-08T09:22:43,561 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#42 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:43,562 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a076ed203726485f9de7d4b8ed0d3863 is 50, key is test_row_0/B:col10/1731057761078/Put/seqid=0 2024-11-08T09:22:43,562 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#43 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:43,562 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/78e026bea673497c955288aed7d80dce is 50, key is test_row_0/A:col10/1731057761078/Put/seqid=0 2024-11-08T09:22:43,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741879_1055 (size=12629) 2024-11-08T09:22:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T09:22:43,602 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:43,603 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-08T09:22:43,606 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T09:22:43,609 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:43,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:43,614 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a076ed203726485f9de7d4b8ed0d3863 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a076ed203726485f9de7d4b8ed0d3863 2024-11-08T09:22:43,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741880_1056 (size=12629) 2024-11-08T09:22:43,624 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into a076ed203726485f9de7d4b8ed0d3863(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:43,624 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:43,624 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=12, startTime=1731057763534; duration=0sec 2024-11-08T09:22:43,625 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:43,625 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:43,625 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:22:43,627 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:22:43,627 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:43,627 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:43,628 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/69a5e35c16bd45dcbde1522409660025, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/55916363ac994d42ae9d2d9cc9c8141b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/56ae784dd5d94ec59919d1cd8e230b4d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a0559add7fe4cb2a84960bc3ca38204] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=47.8 K 2024-11-08T09:22:43,628 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 69a5e35c16bd45dcbde1522409660025, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731057758520 2024-11-08T09:22:43,629 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 55916363ac994d42ae9d2d9cc9c8141b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731057759814 2024-11-08T09:22:43,629 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ae784dd5d94ec59919d1cd8e230b4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057759883 2024-11-08T09:22:43,630 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a0559add7fe4cb2a84960bc3ca38204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731057761078 2024-11-08T09:22:43,645 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:43,646 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5147ce80088f4fd9a0ff0a8624bda41b is 50, key is test_row_0/C:col10/1731057761078/Put/seqid=0 2024-11-08T09:22:43,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741881_1057 (size=12629) 2024-11-08T09:22:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T09:22:43,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T09:22:43,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:43,764 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T09:22:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f26ed9c0cdec47b7913f7163ab679d62 is 50, key is test_row_0/A:col10/1731057762255/Put/seqid=0 2024-11-08T09:22:43,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741882_1058 (size=12151) 2024-11-08T09:22:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T09:22:44,031 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/78e026bea673497c955288aed7d80dce as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/78e026bea673497c955288aed7d80dce 2024-11-08T09:22:44,039 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into 78e026bea673497c955288aed7d80dce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:44,039 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:44,039 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=12, startTime=1731057763534; duration=0sec 2024-11-08T09:22:44,039 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:44,039 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:44,069 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5147ce80088f4fd9a0ff0a8624bda41b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5147ce80088f4fd9a0ff0a8624bda41b 2024-11-08T09:22:44,077 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into 5147ce80088f4fd9a0ff0a8624bda41b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:44,077 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:44,077 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=12, startTime=1731057763534; duration=0sec 2024-11-08T09:22:44,077 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:44,077 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:44,179 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f26ed9c0cdec47b7913f7163ab679d62 2024-11-08T09:22:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/8033915f52874e778aa1993198d6ee3d is 50, key is test_row_0/B:col10/1731057762255/Put/seqid=0 2024-11-08T09:22:44,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741883_1059 (size=12151) 2024-11-08T09:22:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T09:22:44,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057824427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057824427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057824429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057824429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057824430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057824532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057824532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057824541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057824542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057824542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,611 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/8033915f52874e778aa1993198d6ee3d 2024-11-08T09:22:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5a5eb66a1dd24f1c918fe072b3ec7b18 is 50, key is test_row_0/C:col10/1731057762255/Put/seqid=0 2024-11-08T09:22:44,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741884_1060 (size=12151) 2024-11-08T09:22:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T09:22:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057824742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057824742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057824752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057824752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057824752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,041 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5a5eb66a1dd24f1c918fe072b3ec7b18 2024-11-08T09:22:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f26ed9c0cdec47b7913f7163ab679d62 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f26ed9c0cdec47b7913f7163ab679d62 2024-11-08T09:22:45,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057825051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057825051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,059 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f26ed9c0cdec47b7913f7163ab679d62, entries=150, sequenceid=236, filesize=11.9 K 2024-11-08T09:22:45,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/8033915f52874e778aa1993198d6ee3d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8033915f52874e778aa1993198d6ee3d 2024-11-08T09:22:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057825061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057825062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057825062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,067 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8033915f52874e778aa1993198d6ee3d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-08T09:22:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/5a5eb66a1dd24f1c918fe072b3ec7b18 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a5eb66a1dd24f1c918fe072b3ec7b18 2024-11-08T09:22:45,075 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a5eb66a1dd24f1c918fe072b3ec7b18, entries=150, sequenceid=236, filesize=11.9 K 2024-11-08T09:22:45,076 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1312ms, sequenceid=236, compaction requested=false 2024-11-08T09:22:45,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:45,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:45,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-08T09:22:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-08T09:22:45,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T09:22:45,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4690 sec 2024-11-08T09:22:45,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4780 sec 2024-11-08T09:22:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:45,564 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:45,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T09:22:45,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/cd2aa9c31dde43d5882ea31652325ced is 50, key is test_row_0/A:col10/1731057765562/Put/seqid=0 2024-11-08T09:22:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741885_1061 (size=14541) 2024-11-08T09:22:45,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057825615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057825614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057825648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057825648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057825649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T09:22:45,753 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:45,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-08T09:22:45,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057825752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057825752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T09:22:45,756 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:45,757 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:45,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:45,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057825761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057825762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057825762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T09:22:45,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:45,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:45,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:45,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:45,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:45,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057825961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057825962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057825972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057825972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057825972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:45,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/cd2aa9c31dde43d5882ea31652325ced 2024-11-08T09:22:45,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a537f39447bc4c68b69ee1963a467589 is 50, key is test_row_0/B:col10/1731057765562/Put/seqid=0 2024-11-08T09:22:45,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741886_1062 (size=12151) 2024-11-08T09:22:46,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:46,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:46,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T09:22:46,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:46,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:46,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057826271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057826271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057826282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057826282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057826282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T09:22:46,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a537f39447bc4c68b69ee1963a467589 2024-11-08T09:22:46,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/e724f861da734694a4b55fea489bdc31 is 50, key is test_row_0/C:col10/1731057765562/Put/seqid=0 2024-11-08T09:22:46,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741887_1063 (size=12151) 2024-11-08T09:22:46,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057826782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057826782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057826791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057826791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057826792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:46,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/e724f861da734694a4b55fea489bdc31 2024-11-08T09:22:46,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/cd2aa9c31dde43d5882ea31652325ced as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/cd2aa9c31dde43d5882ea31652325ced 2024-11-08T09:22:46,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/cd2aa9c31dde43d5882ea31652325ced, entries=200, sequenceid=256, filesize=14.2 K 2024-11-08T09:22:46,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a537f39447bc4c68b69ee1963a467589 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a537f39447bc4c68b69ee1963a467589 2024-11-08T09:22:46,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a537f39447bc4c68b69ee1963a467589, entries=150, sequenceid=256, filesize=11.9 K 2024-11-08T09:22:46,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/e724f861da734694a4b55fea489bdc31 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/e724f861da734694a4b55fea489bdc31 2024-11-08T09:22:46,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:46,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/e724f861da734694a4b55fea489bdc31, entries=150, sequenceid=256, filesize=11.9 K 2024-11-08T09:22:46,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1278ms, sequenceid=256, compaction requested=true 2024-11-08T09:22:46,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:46,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:46,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:46,843 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:46,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:46,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:46,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:46,843 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:46,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:46,845 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:46,845 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:46,845 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,845 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a076ed203726485f9de7d4b8ed0d3863, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8033915f52874e778aa1993198d6ee3d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a537f39447bc4c68b69ee1963a467589] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.1 K 2024-11-08T09:22:46,846 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:46,846 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:46,846 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,846 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/78e026bea673497c955288aed7d80dce, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f26ed9c0cdec47b7913f7163ab679d62, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/cd2aa9c31dde43d5882ea31652325ced] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=38.4 K 2024-11-08T09:22:46,846 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting a076ed203726485f9de7d4b8ed0d3863, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731057761078 2024-11-08T09:22:46,847 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e026bea673497c955288aed7d80dce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731057761078 2024-11-08T09:22:46,847 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8033915f52874e778aa1993198d6ee3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731057762255 2024-11-08T09:22:46,847 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f26ed9c0cdec47b7913f7163ab679d62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731057762255 2024-11-08T09:22:46,847 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2aa9c31dde43d5882ea31652325ced, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731057764419 2024-11-08T09:22:46,847 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting a537f39447bc4c68b69ee1963a467589, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731057764419 2024-11-08T09:22:46,861 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:46,861 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/39ff885b1c7a430a852aab070c017688 is 50, key is test_row_0/B:col10/1731057765562/Put/seqid=0 2024-11-08T09:22:46,862 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:46,862 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/8541d1dc21c64a0cae741b79b93966b0 is 50, key is test_row_0/A:col10/1731057765562/Put/seqid=0 2024-11-08T09:22:46,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741889_1065 (size=12731) 2024-11-08T09:22:46,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741888_1064 (size=12731) 2024-11-08T09:22:46,884 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/39ff885b1c7a430a852aab070c017688 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/39ff885b1c7a430a852aab070c017688 2024-11-08T09:22:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T09:22:46,892 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into 39ff885b1c7a430a852aab070c017688(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:46,892 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:46,892 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=13, startTime=1731057766843; duration=0sec 2024-11-08T09:22:46,893 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:46,894 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:46,894 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:46,895 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:46,896 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:46,896 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,896 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5147ce80088f4fd9a0ff0a8624bda41b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a5eb66a1dd24f1c918fe072b3ec7b18, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/e724f861da734694a4b55fea489bdc31] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.1 K 2024-11-08T09:22:46,896 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5147ce80088f4fd9a0ff0a8624bda41b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731057761078 2024-11-08T09:22:46,897 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a5eb66a1dd24f1c918fe072b3ec7b18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731057762255 2024-11-08T09:22:46,898 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e724f861da734694a4b55fea489bdc31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731057764419 2024-11-08T09:22:46,908 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:46,909 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/82c1a21390c14a10b7919adbfd942d66 is 50, key is test_row_0/C:col10/1731057765562/Put/seqid=0 2024-11-08T09:22:46,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741890_1066 (size=12731) 2024-11-08T09:22:46,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T09:22:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:46,995 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T09:22:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c4737e3bbad1414bbc981b53da6a298b is 50, key is test_row_0/A:col10/1731057765620/Put/seqid=0 2024-11-08T09:22:47,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741891_1067 (size=12301) 2024-11-08T09:22:47,008 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c4737e3bbad1414bbc981b53da6a298b 2024-11-08T09:22:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/843def7fa5104b37b5dc60e635867fca is 50, key is test_row_0/B:col10/1731057765620/Put/seqid=0 2024-11-08T09:22:47,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741892_1068 (size=12301) 2024-11-08T09:22:47,277 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/8541d1dc21c64a0cae741b79b93966b0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8541d1dc21c64a0cae741b79b93966b0 2024-11-08T09:22:47,283 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into 8541d1dc21c64a0cae741b79b93966b0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:47,283 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:47,283 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=13, startTime=1731057766842; duration=0sec 2024-11-08T09:22:47,284 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:47,284 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:47,320 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/82c1a21390c14a10b7919adbfd942d66 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/82c1a21390c14a10b7919adbfd942d66 2024-11-08T09:22:47,326 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into 82c1a21390c14a10b7919adbfd942d66(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:47,327 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:47,327 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=13, startTime=1731057766843; duration=0sec 2024-11-08T09:22:47,327 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:47,327 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:47,429 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/843def7fa5104b37b5dc60e635867fca 2024-11-08T09:22:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/af5104afc0e9421e85c19a6dd77e1d22 is 50, key is test_row_0/C:col10/1731057765620/Put/seqid=0 2024-11-08T09:22:47,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741893_1069 (size=12301) 2024-11-08T09:22:47,455 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/af5104afc0e9421e85c19a6dd77e1d22 2024-11-08T09:22:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c4737e3bbad1414bbc981b53da6a298b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c4737e3bbad1414bbc981b53da6a298b 2024-11-08T09:22:47,470 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c4737e3bbad1414bbc981b53da6a298b, entries=150, sequenceid=275, filesize=12.0 K 2024-11-08T09:22:47,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/843def7fa5104b37b5dc60e635867fca as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/843def7fa5104b37b5dc60e635867fca 2024-11-08T09:22:47,481 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/843def7fa5104b37b5dc60e635867fca, entries=150, sequenceid=275, filesize=12.0 K 2024-11-08T09:22:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/af5104afc0e9421e85c19a6dd77e1d22 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/af5104afc0e9421e85c19a6dd77e1d22 2024-11-08T09:22:47,492 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/af5104afc0e9421e85c19a6dd77e1d22, entries=150, sequenceid=275, filesize=12.0 K 2024-11-08T09:22:47,494 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 6f50fcf560ee3a7bd187b06bd37f77bb in 499ms, sequenceid=275, compaction requested=false 2024-11-08T09:22:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-08T09:22:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-08T09:22:47,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-08T09:22:47,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-08T09:22:47,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7470 sec 2024-11-08T09:22:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:47,825 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:47,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:22:47,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f2f77942798b49f9b8451c4d877687d2 is 50, key is test_row_0/A:col10/1731057767824/Put/seqid=0 2024-11-08T09:22:47,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741894_1070 (size=19621) 2024-11-08T09:22:47,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057827859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057827862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057827862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057827864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057827864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T09:22:47,903 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:47,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-08T09:22:47,907 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T09:22:47,908 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:47,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:47,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057827971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057827972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057827972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057827972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:47,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057827981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:47,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:47,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T09:22:48,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:48,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057828181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057828181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057828182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057828182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057828191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T09:22:48,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f2f77942798b49f9b8451c4d877687d2 2024-11-08T09:22:48,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/915900c8120c409798f1b9216bf13e4f is 50, key is test_row_0/B:col10/1731057767824/Put/seqid=0 2024-11-08T09:22:48,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741895_1071 (size=12301) 2024-11-08T09:22:48,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057828492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057828492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057828492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057828492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057828501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:48,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:48,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:48,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T09:22:48,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/915900c8120c409798f1b9216bf13e4f 2024-11-08T09:22:48,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:48,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/be7ebc2f813d4baeab522dcbb4d6875f is 50, key is test_row_0/C:col10/1731057767824/Put/seqid=0 2024-11-08T09:22:48,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741896_1072 (size=12301) 2024-11-08T09:22:48,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:48,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:48,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:48,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:49,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057829002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:49,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:49,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057829002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:49,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057829002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057829012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:49,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057829012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:49,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:49,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:49,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T09:22:49,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/be7ebc2f813d4baeab522dcbb4d6875f 2024-11-08T09:22:49,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f2f77942798b49f9b8451c4d877687d2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f2f77942798b49f9b8451c4d877687d2 2024-11-08T09:22:49,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f2f77942798b49f9b8451c4d877687d2, entries=300, sequenceid=289, filesize=19.2 K 2024-11-08T09:22:49,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/915900c8120c409798f1b9216bf13e4f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/915900c8120c409798f1b9216bf13e4f 2024-11-08T09:22:49,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/915900c8120c409798f1b9216bf13e4f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T09:22:49,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/be7ebc2f813d4baeab522dcbb4d6875f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/be7ebc2f813d4baeab522dcbb4d6875f 2024-11-08T09:22:49,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/be7ebc2f813d4baeab522dcbb4d6875f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T09:22:49,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1308ms, sequenceid=289, compaction requested=true 2024-11-08T09:22:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:49,133 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:49,133 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:49,134 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:49,135 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:49,135 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:49,135 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:49,135 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:49,135 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:49,135 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8541d1dc21c64a0cae741b79b93966b0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c4737e3bbad1414bbc981b53da6a298b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f2f77942798b49f9b8451c4d877687d2] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=43.6 K 2024-11-08T09:22:49,135 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/39ff885b1c7a430a852aab070c017688, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/843def7fa5104b37b5dc60e635867fca, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/915900c8120c409798f1b9216bf13e4f] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.5 K 2024-11-08T09:22:49,135 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8541d1dc21c64a0cae741b79b93966b0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731057764419 2024-11-08T09:22:49,136 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ff885b1c7a430a852aab070c017688, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731057764419 2024-11-08T09:22:49,136 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4737e3bbad1414bbc981b53da6a298b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731057765615 2024-11-08T09:22:49,136 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 843def7fa5104b37b5dc60e635867fca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731057765615 2024-11-08T09:22:49,137 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f77942798b49f9b8451c4d877687d2, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057767796 2024-11-08T09:22:49,137 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 915900c8120c409798f1b9216bf13e4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057767817 2024-11-08T09:22:49,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T09:22:49,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:49,139 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:22:49,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c51f1958472c4b7aa0a0795e1c8dd542 is 50, key is test_row_0/A:col10/1731057767862/Put/seqid=0 2024-11-08T09:22:49,149 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:49,150 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/e24fc69c922c4cb9ad37524b54f13ab1 is 50, key is test_row_0/A:col10/1731057767824/Put/seqid=0 2024-11-08T09:22:49,151 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:49,152 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/79193eca0963457da80fb99664063bfa is 50, key is test_row_0/B:col10/1731057767824/Put/seqid=0 2024-11-08T09:22:49,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741897_1073 (size=12301) 2024-11-08T09:22:49,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741898_1074 (size=12983) 2024-11-08T09:22:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741899_1075 (size=12983) 2024-11-08T09:22:49,181 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/79193eca0963457da80fb99664063bfa as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/79193eca0963457da80fb99664063bfa 2024-11-08T09:22:49,189 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/e24fc69c922c4cb9ad37524b54f13ab1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e24fc69c922c4cb9ad37524b54f13ab1 2024-11-08T09:22:49,189 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into 79193eca0963457da80fb99664063bfa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:49,189 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:49,189 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=13, startTime=1731057769133; duration=0sec 2024-11-08T09:22:49,190 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:49,190 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:49,190 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:49,191 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:49,191 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:49,191 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:49,191 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/82c1a21390c14a10b7919adbfd942d66, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/af5104afc0e9421e85c19a6dd77e1d22, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/be7ebc2f813d4baeab522dcbb4d6875f] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.5 K 2024-11-08T09:22:49,192 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 82c1a21390c14a10b7919adbfd942d66, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731057764419 2024-11-08T09:22:49,192 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting af5104afc0e9421e85c19a6dd77e1d22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731057765615 2024-11-08T09:22:49,193 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting be7ebc2f813d4baeab522dcbb4d6875f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057767817 2024-11-08T09:22:49,198 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into e24fc69c922c4cb9ad37524b54f13ab1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:49,198 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:49,199 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=13, startTime=1731057769133; duration=0sec 2024-11-08T09:22:49,199 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:49,199 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:49,203 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:49,204 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/b7a2b9bcf0ce4590a70bb8ade82537e3 is 50, key is test_row_0/C:col10/1731057767824/Put/seqid=0 2024-11-08T09:22:49,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741900_1076 (size=12983) 2024-11-08T09:22:49,559 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c51f1958472c4b7aa0a0795e1c8dd542 2024-11-08T09:22:49,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/8d6c14739cd24054ba01f214fde5c8bc is 50, key is test_row_0/B:col10/1731057767862/Put/seqid=0 2024-11-08T09:22:49,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741901_1077 (size=12301) 2024-11-08T09:22:49,580 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/8d6c14739cd24054ba01f214fde5c8bc 2024-11-08T09:22:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/45650774b81d4a309f669c2b92b191b3 is 50, key is test_row_0/C:col10/1731057767862/Put/seqid=0 2024-11-08T09:22:49,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741902_1078 (size=12301) 2024-11-08T09:22:49,623 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/b7a2b9bcf0ce4590a70bb8ade82537e3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/b7a2b9bcf0ce4590a70bb8ade82537e3 2024-11-08T09:22:49,630 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into b7a2b9bcf0ce4590a70bb8ade82537e3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:49,630 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:49,631 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=13, startTime=1731057769133; duration=0sec 2024-11-08T09:22:49,631 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:49,631 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:50,001 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/45650774b81d4a309f669c2b92b191b3 2024-11-08T09:22:50,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/c51f1958472c4b7aa0a0795e1c8dd542 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c51f1958472c4b7aa0a0795e1c8dd542 2024-11-08T09:22:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:50,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:50,015 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c51f1958472c4b7aa0a0795e1c8dd542, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T09:22:50,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/8d6c14739cd24054ba01f214fde5c8bc as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8d6c14739cd24054ba01f214fde5c8bc 2024-11-08T09:22:50,024 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8d6c14739cd24054ba01f214fde5c8bc, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T09:22:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/45650774b81d4a309f669c2b92b191b3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/45650774b81d4a309f669c2b92b191b3 2024-11-08T09:22:50,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057830025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057830026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057830027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057830028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057830031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,032 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/45650774b81d4a309f669c2b92b191b3, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T09:22:50,034 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 6f50fcf560ee3a7bd187b06bd37f77bb in 894ms, sequenceid=314, compaction requested=false 2024-11-08T09:22:50,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:50,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-08T09:22:50,035 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in 6f50fcf560ee3a7bd187b06bd37f77bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:22:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-08T09:22:50,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T09:22:50,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1300 sec 2024-11-08T09:22:50,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1380 sec 2024-11-08T09:22:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T09:22:50,052 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:50,052 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:22:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-08T09:22:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T09:22:50,055 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:22:50,057 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:22:50,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:22:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:50,133 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:50,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T09:22:50,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f55a6e726cc24644a826b153b84465d0 is 50, key is test_row_0/A:col10/1731057770132/Put/seqid=0 2024-11-08T09:22:50,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741903_1079 (size=14741) 2024-11-08T09:22:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T09:22:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057830193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057830193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057830194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057830199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057830301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057830301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057830301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057830312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T09:22:50,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057830512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057830512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057830512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:50,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057830521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f55a6e726cc24644a826b153b84465d0 2024-11-08T09:22:50,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/21889d853d9c41079572ce322e76f3a4 is 50, key is test_row_0/B:col10/1731057770132/Put/seqid=0 2024-11-08T09:22:50,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741904_1080 (size=12301) 2024-11-08T09:22:50,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:50,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:50,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T09:22:50,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057830821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057830822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057830822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057830831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:50,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/21889d853d9c41079572ce322e76f3a4 2024-11-08T09:22:50,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/f2d1e16cac0549458741fa45cdaf632e is 50, key is test_row_0/C:col10/1731057770132/Put/seqid=0 2024-11-08T09:22:50,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:50,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741905_1081 (size=12301) 2024-11-08T09:22:51,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T09:22:51,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:51,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057831332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057831332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:51,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057831332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:51,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057831341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:51,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:22:51,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/f2d1e16cac0549458741fa45cdaf632e 2024-11-08T09:22:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/f55a6e726cc24644a826b153b84465d0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f55a6e726cc24644a826b153b84465d0 2024-11-08T09:22:51,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f55a6e726cc24644a826b153b84465d0, entries=200, sequenceid=331, filesize=14.4 K 2024-11-08T09:22:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/21889d853d9c41079572ce322e76f3a4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/21889d853d9c41079572ce322e76f3a4 2024-11-08T09:22:51,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/21889d853d9c41079572ce322e76f3a4, entries=150, sequenceid=331, filesize=12.0 K 2024-11-08T09:22:51,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/f2d1e16cac0549458741fa45cdaf632e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/f2d1e16cac0549458741fa45cdaf632e 2024-11-08T09:22:51,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/f2d1e16cac0549458741fa45cdaf632e, entries=150, sequenceid=331, filesize=12.0 K 2024-11-08T09:22:51,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1270ms, sequenceid=331, compaction requested=true 2024-11-08T09:22:51,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:51,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:51,405 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:51,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:51,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:51,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:51,405 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:51,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:51,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:51,406 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:51,406 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:51,406 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:51,406 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:51,406 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,406 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,406 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e24fc69c922c4cb9ad37524b54f13ab1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c51f1958472c4b7aa0a0795e1c8dd542, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f55a6e726cc24644a826b153b84465d0] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=39.1 K 2024-11-08T09:22:51,406 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/79193eca0963457da80fb99664063bfa, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8d6c14739cd24054ba01f214fde5c8bc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/21889d853d9c41079572ce322e76f3a4] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.7 K 2024-11-08T09:22:51,407 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 79193eca0963457da80fb99664063bfa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057767817 2024-11-08T09:22:51,407 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e24fc69c922c4cb9ad37524b54f13ab1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057767817 2024-11-08T09:22:51,407 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6c14739cd24054ba01f214fde5c8bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731057767855 2024-11-08T09:22:51,407 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51f1958472c4b7aa0a0795e1c8dd542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731057767855 2024-11-08T09:22:51,407 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 21889d853d9c41079572ce322e76f3a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731057770025 2024-11-08T09:22:51,407 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55a6e726cc24644a826b153b84465d0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731057770023 2024-11-08T09:22:51,415 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:51,416 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/d90f70d97abc44378b5045b1da2559aa is 50, key is test_row_0/A:col10/1731057770132/Put/seqid=0 2024-11-08T09:22:51,417 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:51,418 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/67677885f0cd48b3b1b71eb075476cd7 is 50, key is test_row_0/B:col10/1731057770132/Put/seqid=0 2024-11-08T09:22:51,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741906_1082 (size=13085) 2024-11-08T09:22:51,434 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/d90f70d97abc44378b5045b1da2559aa as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d90f70d97abc44378b5045b1da2559aa 2024-11-08T09:22:51,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741907_1083 (size=13085) 2024-11-08T09:22:51,442 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into d90f70d97abc44378b5045b1da2559aa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:51,442 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:51,442 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=13, startTime=1731057771405; duration=0sec 2024-11-08T09:22:51,443 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:51,443 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:51,443 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:51,445 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:51,445 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:51,445 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,445 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/b7a2b9bcf0ce4590a70bb8ade82537e3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/45650774b81d4a309f669c2b92b191b3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/f2d1e16cac0549458741fa45cdaf632e] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.7 K 2024-11-08T09:22:51,446 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a2b9bcf0ce4590a70bb8ade82537e3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057767817 2024-11-08T09:22:51,446 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45650774b81d4a309f669c2b92b191b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731057767855 2024-11-08T09:22:51,447 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/67677885f0cd48b3b1b71eb075476cd7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/67677885f0cd48b3b1b71eb075476cd7 2024-11-08T09:22:51,447 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2d1e16cac0549458741fa45cdaf632e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731057770025 2024-11-08T09:22:51,459 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into 67677885f0cd48b3b1b71eb075476cd7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:51,460 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:51,460 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=13, startTime=1731057771405; duration=0sec 2024-11-08T09:22:51,460 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:51,460 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:51,462 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:51,462 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/1693aff22ab94eb18f641e2e97e1b65f is 50, key is test_row_0/C:col10/1731057770132/Put/seqid=0 2024-11-08T09:22:51,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741908_1084 (size=13085) 2024-11-08T09:22:51,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T09:22:51,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:51,509 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T09:22:51,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/6615e363192c439f8178f2b0f087163b is 50, key is test_row_0/A:col10/1731057770191/Put/seqid=0 2024-11-08T09:22:51,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741909_1085 (size=12301) 2024-11-08T09:22:51,876 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/1693aff22ab94eb18f641e2e97e1b65f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1693aff22ab94eb18f641e2e97e1b65f 2024-11-08T09:22:51,883 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into 1693aff22ab94eb18f641e2e97e1b65f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:51,883 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:51,883 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=13, startTime=1731057771405; duration=0sec 2024-11-08T09:22:51,883 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:51,883 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:51,920 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/6615e363192c439f8178f2b0f087163b 2024-11-08T09:22:51,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a05963225b2c4cdcaafb5aaf8a7d6ad0 is 50, key is test_row_0/B:col10/1731057770191/Put/seqid=0 2024-11-08T09:22:51,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741910_1086 (size=12301) 2024-11-08T09:22:51,942 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a05963225b2c4cdcaafb5aaf8a7d6ad0 2024-11-08T09:22:51,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/83fd6703c36d485dbc474114c5309282 is 50, key is test_row_0/C:col10/1731057770191/Put/seqid=0 2024-11-08T09:22:51,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741911_1087 (size=12301) 2024-11-08T09:22:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. as already flushing 2024-11-08T09:22:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057832094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057832202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T09:22:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50666 deadline: 1731057832341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,344 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in 6f50fcf560ee3a7bd187b06bd37f77bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:22:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50694 deadline: 1731057832341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,347 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 6f50fcf560ee3a7bd187b06bd37f77bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:22:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50718 deadline: 1731057832352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50716 deadline: 1731057832352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,354 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 6f50fcf560ee3a7bd187b06bd37f77bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:22:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,355 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 6f50fcf560ee3a7bd187b06bd37f77bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:22:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,356 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/83fd6703c36d485dbc474114c5309282 2024-11-08T09:22:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/6615e363192c439f8178f2b0f087163b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/6615e363192c439f8178f2b0f087163b 2024-11-08T09:22:52,368 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/6615e363192c439f8178f2b0f087163b, entries=150, sequenceid=355, filesize=12.0 K 2024-11-08T09:22:52,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/a05963225b2c4cdcaafb5aaf8a7d6ad0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a05963225b2c4cdcaafb5aaf8a7d6ad0 2024-11-08T09:22:52,374 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a05963225b2c4cdcaafb5aaf8a7d6ad0, entries=150, sequenceid=355, filesize=12.0 K 2024-11-08T09:22:52,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/83fd6703c36d485dbc474114c5309282 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/83fd6703c36d485dbc474114c5309282 2024-11-08T09:22:52,380 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/83fd6703c36d485dbc474114c5309282, entries=150, sequenceid=355, filesize=12.0 K 2024-11-08T09:22:52,382 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6f50fcf560ee3a7bd187b06bd37f77bb in 872ms, sequenceid=355, compaction requested=false 2024-11-08T09:22:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-08T09:22:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-08T09:22:52,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T09:22:52,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-11-08T09:22:52,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.3340 sec 2024-11-08T09:22:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:52,415 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:52,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T09:22:52,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/e4cb1f7ebd074d5bac252cb07f6b2cb3 is 50, key is test_row_0/A:col10/1731057772414/Put/seqid=0 2024-11-08T09:22:52,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741912_1088 (size=14741) 2024-11-08T09:22:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057832475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057832581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057832792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:52,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/e4cb1f7ebd074d5bac252cb07f6b2cb3 2024-11-08T09:22:52,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/e680eab4dc3748dfbe3d58df2a6312ee is 50, key is test_row_0/B:col10/1731057772414/Put/seqid=0 2024-11-08T09:22:52,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741913_1089 (size=12301) 2024-11-08T09:22:52,978 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-08T09:22:52,978 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:52,980 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-08T09:22:52,980 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:52,981 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-08T09:22:52,981 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:52,981 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-08T09:22:52,981 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:52,983 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,983 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,983 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,983 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,983 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:52,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:52,983 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,984 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:52,984 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:52,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057833102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:53,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/e680eab4dc3748dfbe3d58df2a6312ee 2024-11-08T09:22:53,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/3abf891e83394768bd1d909c92154954 is 50, key is test_row_0/C:col10/1731057772414/Put/seqid=0 2024-11-08T09:22:53,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741914_1090 (size=12301) 2024-11-08T09:22:53,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:22:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1731057833612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 2024-11-08T09:22:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:22:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:22:53,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/3abf891e83394768bd1d909c92154954 2024-11-08T09:22:53,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/e4cb1f7ebd074d5bac252cb07f6b2cb3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e4cb1f7ebd074d5bac252cb07f6b2cb3 2024-11-08T09:22:53,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e4cb1f7ebd074d5bac252cb07f6b2cb3, entries=200, sequenceid=371, filesize=14.4 K 2024-11-08T09:22:53,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/e680eab4dc3748dfbe3d58df2a6312ee as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/e680eab4dc3748dfbe3d58df2a6312ee 2024-11-08T09:22:53,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/e680eab4dc3748dfbe3d58df2a6312ee, entries=150, sequenceid=371, filesize=12.0 K 2024-11-08T09:22:53,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/3abf891e83394768bd1d909c92154954 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3abf891e83394768bd1d909c92154954 2024-11-08T09:22:53,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3abf891e83394768bd1d909c92154954, entries=150, sequenceid=371, filesize=12.0 K 2024-11-08T09:22:53,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1301ms, sequenceid=371, compaction requested=true 2024-11-08T09:22:53,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:22:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:53,717 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:22:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:53,717 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f50fcf560ee3a7bd187b06bd37f77bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:22:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:53,718 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:53,718 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:53,718 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/B is initiating minor compaction (all files) 2024-11-08T09:22:53,718 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/A is initiating minor compaction (all files) 2024-11-08T09:22:53,718 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/B in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:53,718 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/A in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:53,718 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/67677885f0cd48b3b1b71eb075476cd7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a05963225b2c4cdcaafb5aaf8a7d6ad0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/e680eab4dc3748dfbe3d58df2a6312ee] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.8 K 2024-11-08T09:22:53,718 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d90f70d97abc44378b5045b1da2559aa, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/6615e363192c439f8178f2b0f087163b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e4cb1f7ebd074d5bac252cb07f6b2cb3] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=39.2 K 2024-11-08T09:22:53,718 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 67677885f0cd48b3b1b71eb075476cd7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731057770025 2024-11-08T09:22:53,718 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90f70d97abc44378b5045b1da2559aa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731057770025 2024-11-08T09:22:53,719 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a05963225b2c4cdcaafb5aaf8a7d6ad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731057770186 2024-11-08T09:22:53,719 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6615e363192c439f8178f2b0f087163b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731057770186 2024-11-08T09:22:53,719 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e680eab4dc3748dfbe3d58df2a6312ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731057772089 2024-11-08T09:22:53,719 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4cb1f7ebd074d5bac252cb07f6b2cb3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731057772082 2024-11-08T09:22:53,728 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:53,728 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:53,729 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/93fc873ef42348cd8bcf2a8c154d3400 is 50, key is test_row_0/B:col10/1731057772414/Put/seqid=0 2024-11-08T09:22:53,729 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/b87f8da825ba472d9fd10d47a73f737f is 50, key is test_row_0/A:col10/1731057772414/Put/seqid=0 2024-11-08T09:22:53,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741915_1091 (size=13187) 2024-11-08T09:22:53,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741916_1092 (size=13187) 2024-11-08T09:22:54,145 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/93fc873ef42348cd8bcf2a8c154d3400 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/93fc873ef42348cd8bcf2a8c154d3400 2024-11-08T09:22:54,145 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/b87f8da825ba472d9fd10d47a73f737f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b87f8da825ba472d9fd10d47a73f737f 2024-11-08T09:22:54,151 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/A of 6f50fcf560ee3a7bd187b06bd37f77bb into b87f8da825ba472d9fd10d47a73f737f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:54,151 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/B of 6f50fcf560ee3a7bd187b06bd37f77bb into 93fc873ef42348cd8bcf2a8c154d3400(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:54,151 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:54,151 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:54,151 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/A, priority=13, startTime=1731057773716; duration=0sec 2024-11-08T09:22:54,151 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/B, priority=13, startTime=1731057773717; duration=0sec 2024-11-08T09:22:54,151 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:54,151 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:B 2024-11-08T09:22:54,151 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:22:54,152 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:A 2024-11-08T09:22:54,152 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:22:54,153 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:22:54,153 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 6f50fcf560ee3a7bd187b06bd37f77bb/C is initiating minor compaction (all files) 2024-11-08T09:22:54,153 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f50fcf560ee3a7bd187b06bd37f77bb/C in TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:54,153 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1693aff22ab94eb18f641e2e97e1b65f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/83fd6703c36d485dbc474114c5309282, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3abf891e83394768bd1d909c92154954] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp, totalSize=36.8 K 2024-11-08T09:22:54,154 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 1693aff22ab94eb18f641e2e97e1b65f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731057770025 2024-11-08T09:22:54,154 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 83fd6703c36d485dbc474114c5309282, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731057770186 2024-11-08T09:22:54,154 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 3abf891e83394768bd1d909c92154954, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731057772089 2024-11-08T09:22:54,162 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f50fcf560ee3a7bd187b06bd37f77bb#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:22:54,163 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/1fb852928d8c4152b8887064f2c7021b is 50, key is test_row_0/C:col10/1731057772414/Put/seqid=0 2024-11-08T09:22:54,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741917_1093 (size=13187) 2024-11-08T09:22:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T09:22:54,232 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:22:54,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T09:22:54,363 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-08T09:22:54,363 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-08T09:22:54,364 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:54,364 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:54,364 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,364 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,364 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,364 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:54,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:54,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/d5828fb7b64c4e2ab20ad11d019a9ba8 is 50, key is test_row_0/A:col10/1731057772461/Put/seqid=0 2024-11-08T09:22:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741918_1094 (size=12301) 2024-11-08T09:22:54,373 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-08T09:22:54,373 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-08T09:22:54,373 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:54,373 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:54,373 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,373 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,373 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,373 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:54,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:54,580 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/1fb852928d8c4152b8887064f2c7021b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1fb852928d8c4152b8887064f2c7021b 2024-11-08T09:22:54,589 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f50fcf560ee3a7bd187b06bd37f77bb/C of 6f50fcf560ee3a7bd187b06bd37f77bb into 1fb852928d8c4152b8887064f2c7021b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:22:54,589 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:54,590 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., storeName=6f50fcf560ee3a7bd187b06bd37f77bb/C, priority=13, startTime=1731057773717; duration=0sec 2024-11-08T09:22:54,590 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:22:54,590 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f50fcf560ee3a7bd187b06bd37f77bb:C 2024-11-08T09:22:54,623 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-08T09:22:54,623 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:22:54,623 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,624 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:54,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T09:22:54,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-08T09:22:54,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-08T09:22:54,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-08T09:22:54,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-08T09:22:54,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-08T09:22:54,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T09:22:54,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4232 2024-11-08T09:22:54,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4123 2024-11-08T09:22:54,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T09:22:54,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1875 2024-11-08T09:22:54,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5617 rows 2024-11-08T09:22:54,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1860 2024-11-08T09:22:54,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5579 rows 2024-11-08T09:22:54,625 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:22:54,625 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:22:54,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:22:54,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:22:54,631 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T09:22:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T09:22:54,640 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057774639"}]},"ts":"1731057774639"} 2024-11-08T09:22:54,642 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T09:22:54,642 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T09:22:54,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:22:54,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, UNASSIGN}] 2024-11-08T09:22:54,649 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, UNASSIGN 2024-11-08T09:22:54,651 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6f50fcf560ee3a7bd187b06bd37f77bb, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:22:54,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T09:22:54,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, UNASSIGN because future has completed 2024-11-08T09:22:54,654 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:22:54,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:22:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T09:22:54,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/d5828fb7b64c4e2ab20ad11d019a9ba8 2024-11-08T09:22:54,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/4e93abddd82d42b3a82047552e7839a1 is 50, key is test_row_0/B:col10/1731057772461/Put/seqid=0 2024-11-08T09:22:54,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741919_1095 (size=12301) 2024-11-08T09:22:54,811 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:54,811 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:22:54,812 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 6f50fcf560ee3a7bd187b06bd37f77bb, disabling compactions & flushes 2024-11-08T09:22:54,812 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T09:22:55,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/4e93abddd82d42b3a82047552e7839a1 2024-11-08T09:22:55,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/eafa8df5f6d2491fb1552ef764446606 is 50, key is test_row_0/C:col10/1731057772461/Put/seqid=0 2024-11-08T09:22:55,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741920_1096 (size=12301) 2024-11-08T09:22:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T09:22:55,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/eafa8df5f6d2491fb1552ef764446606 2024-11-08T09:22:55,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/d5828fb7b64c4e2ab20ad11d019a9ba8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d5828fb7b64c4e2ab20ad11d019a9ba8 2024-11-08T09:22:55,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d5828fb7b64c4e2ab20ad11d019a9ba8, entries=150, sequenceid=396, filesize=12.0 K 2024-11-08T09:22:55,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/4e93abddd82d42b3a82047552e7839a1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4e93abddd82d42b3a82047552e7839a1 2024-11-08T09:22:55,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4e93abddd82d42b3a82047552e7839a1, entries=150, sequenceid=396, filesize=12.0 K 2024-11-08T09:22:55,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/eafa8df5f6d2491fb1552ef764446606 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eafa8df5f6d2491fb1552ef764446606 2024-11-08T09:22:55,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eafa8df5f6d2491fb1552ef764446606, entries=150, sequenceid=396, filesize=12.0 K 2024-11-08T09:22:55,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=20.13 KB/20610 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1286ms, sequenceid=396, compaction requested=false 2024-11-08T09:22:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f50fcf560ee3a7bd187b06bd37f77bb: 2024-11-08T09:22:55,650 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:55,650 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:55,650 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. after waiting 0 ms 2024-11-08T09:22:55,650 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:55,650 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 6f50fcf560ee3a7bd187b06bd37f77bb 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T09:22:55,654 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/742be13d1e1041788f280e54bbd04b1c is 50, key is test_row_0/A:col10/1731057774371/Put/seqid=0 2024-11-08T09:22:55,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741921_1097 (size=9857) 2024-11-08T09:22:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T09:22:56,058 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/742be13d1e1041788f280e54bbd04b1c 2024-11-08T09:22:56,067 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/388684b52a984a42b3029e4705e8f428 is 50, key is test_row_0/B:col10/1731057774371/Put/seqid=0 2024-11-08T09:22:56,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741922_1098 (size=9857) 2024-11-08T09:22:56,473 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/388684b52a984a42b3029e4705e8f428 2024-11-08T09:22:56,533 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/021ea7ea204b48379da48f5e6403587b is 50, key is test_row_0/C:col10/1731057774371/Put/seqid=0 2024-11-08T09:22:56,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741923_1099 (size=9857) 2024-11-08T09:22:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T09:22:56,941 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/021ea7ea204b48379da48f5e6403587b 2024-11-08T09:22:56,951 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/A/742be13d1e1041788f280e54bbd04b1c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/742be13d1e1041788f280e54bbd04b1c 2024-11-08T09:22:56,958 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/742be13d1e1041788f280e54bbd04b1c, entries=100, sequenceid=403, filesize=9.6 K 2024-11-08T09:22:56,959 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/B/388684b52a984a42b3029e4705e8f428 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/388684b52a984a42b3029e4705e8f428 2024-11-08T09:22:56,964 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/388684b52a984a42b3029e4705e8f428, entries=100, sequenceid=403, filesize=9.6 K 2024-11-08T09:22:56,965 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/.tmp/C/021ea7ea204b48379da48f5e6403587b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/021ea7ea204b48379da48f5e6403587b 2024-11-08T09:22:56,969 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/021ea7ea204b48379da48f5e6403587b, entries=100, sequenceid=403, filesize=9.6 K 2024-11-08T09:22:56,970 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1320ms, sequenceid=403, compaction requested=true 2024-11-08T09:22:56,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/0faa37d5a979434a908cb43edd396fc7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/de692b97388f46c182de0cf28fcd98e2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/3f34bf8fb27a4da7975011bc48c82ff1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/567328cd584e455480f869ecc02cae16, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/5f46710c08a944a28ed49dfcacec0e0a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/39f73fb4bceb475e9d56c8dd634cfece, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7431cfae884b4b4992651b98f754ced4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/36df0bc47a26424aa09c1576435c5bb5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b3db1c0e00a24159b439a1d6ca8f983f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8f0288bb622244b6b53d28b2294996bf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/eae161419faf4a6a9e7f0752ea150977, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7c8486678c294cc9a583854f280e0f6d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/ba75d0df22214e249a65308bee5f4fbb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/78e026bea673497c955288aed7d80dce, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c07bd21e4f1f4ca1b7c2f081f32a667c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f26ed9c0cdec47b7913f7163ab679d62, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/cd2aa9c31dde43d5882ea31652325ced, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8541d1dc21c64a0cae741b79b93966b0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c4737e3bbad1414bbc981b53da6a298b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f2f77942798b49f9b8451c4d877687d2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e24fc69c922c4cb9ad37524b54f13ab1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c51f1958472c4b7aa0a0795e1c8dd542, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f55a6e726cc24644a826b153b84465d0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d90f70d97abc44378b5045b1da2559aa, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/6615e363192c439f8178f2b0f087163b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e4cb1f7ebd074d5bac252cb07f6b2cb3] to archive 2024-11-08T09:22:56,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:22:56,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/0faa37d5a979434a908cb43edd396fc7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/0faa37d5a979434a908cb43edd396fc7 2024-11-08T09:22:56,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/de692b97388f46c182de0cf28fcd98e2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/de692b97388f46c182de0cf28fcd98e2 2024-11-08T09:22:56,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/3f34bf8fb27a4da7975011bc48c82ff1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/3f34bf8fb27a4da7975011bc48c82ff1 2024-11-08T09:22:56,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/567328cd584e455480f869ecc02cae16 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/567328cd584e455480f869ecc02cae16 2024-11-08T09:22:56,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/5f46710c08a944a28ed49dfcacec0e0a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/5f46710c08a944a28ed49dfcacec0e0a 2024-11-08T09:22:56,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/39f73fb4bceb475e9d56c8dd634cfece to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/39f73fb4bceb475e9d56c8dd634cfece 2024-11-08T09:22:56,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7431cfae884b4b4992651b98f754ced4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7431cfae884b4b4992651b98f754ced4 2024-11-08T09:22:56,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/36df0bc47a26424aa09c1576435c5bb5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/36df0bc47a26424aa09c1576435c5bb5 2024-11-08T09:22:56,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b3db1c0e00a24159b439a1d6ca8f983f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b3db1c0e00a24159b439a1d6ca8f983f 2024-11-08T09:22:56,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8f0288bb622244b6b53d28b2294996bf to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8f0288bb622244b6b53d28b2294996bf 2024-11-08T09:22:56,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/eae161419faf4a6a9e7f0752ea150977 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/eae161419faf4a6a9e7f0752ea150977 2024-11-08T09:22:56,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7c8486678c294cc9a583854f280e0f6d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/7c8486678c294cc9a583854f280e0f6d 2024-11-08T09:22:56,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/ba75d0df22214e249a65308bee5f4fbb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/ba75d0df22214e249a65308bee5f4fbb 2024-11-08T09:22:57,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/78e026bea673497c955288aed7d80dce to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/78e026bea673497c955288aed7d80dce 2024-11-08T09:22:57,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c07bd21e4f1f4ca1b7c2f081f32a667c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c07bd21e4f1f4ca1b7c2f081f32a667c 2024-11-08T09:22:57,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f26ed9c0cdec47b7913f7163ab679d62 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f26ed9c0cdec47b7913f7163ab679d62 2024-11-08T09:22:57,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/cd2aa9c31dde43d5882ea31652325ced to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/cd2aa9c31dde43d5882ea31652325ced 2024-11-08T09:22:57,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8541d1dc21c64a0cae741b79b93966b0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/8541d1dc21c64a0cae741b79b93966b0 2024-11-08T09:22:57,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c4737e3bbad1414bbc981b53da6a298b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c4737e3bbad1414bbc981b53da6a298b 2024-11-08T09:22:57,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f2f77942798b49f9b8451c4d877687d2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f2f77942798b49f9b8451c4d877687d2 2024-11-08T09:22:57,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e24fc69c922c4cb9ad37524b54f13ab1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e24fc69c922c4cb9ad37524b54f13ab1 2024-11-08T09:22:57,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c51f1958472c4b7aa0a0795e1c8dd542 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/c51f1958472c4b7aa0a0795e1c8dd542 2024-11-08T09:22:57,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f55a6e726cc24644a826b153b84465d0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/f55a6e726cc24644a826b153b84465d0 2024-11-08T09:22:57,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d90f70d97abc44378b5045b1da2559aa to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d90f70d97abc44378b5045b1da2559aa 2024-11-08T09:22:57,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/6615e363192c439f8178f2b0f087163b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/6615e363192c439f8178f2b0f087163b 2024-11-08T09:22:57,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e4cb1f7ebd074d5bac252cb07f6b2cb3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/e4cb1f7ebd074d5bac252cb07f6b2cb3 2024-11-08T09:22:57,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a1b991f93d374902b6afc53e611f52f5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/607ffacc1d704c75813a30376a6f268e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/153c276385c8439686a59d44c0d221fb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/2ce2a7fcd01b41d696d646157d55b288, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/882d3ed4dd124538ae861a304f12c85c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/61074a6545be46439b919fa4bd4f3775, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/c4ab6bda3c9d417ca380bc583602ded5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/389517c9303048d787abbe2db386fdd2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac3d698795564012a0a2253a6b522339, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4246eb446abf41e8964cc029cf6d8fd7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/561fc9ac67644c58b9534ff8e6c36d9a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac59503d178342e99e01c6d6ddfa5b5e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/7ebd68d304184dd8879086c5a5152ec2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a076ed203726485f9de7d4b8ed0d3863, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/5411d6e072b1456091b0ed7666b1da22, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8033915f52874e778aa1993198d6ee3d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/39ff885b1c7a430a852aab070c017688, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a537f39447bc4c68b69ee1963a467589, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/843def7fa5104b37b5dc60e635867fca, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/79193eca0963457da80fb99664063bfa, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/915900c8120c409798f1b9216bf13e4f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8d6c14739cd24054ba01f214fde5c8bc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/67677885f0cd48b3b1b71eb075476cd7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/21889d853d9c41079572ce322e76f3a4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a05963225b2c4cdcaafb5aaf8a7d6ad0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/e680eab4dc3748dfbe3d58df2a6312ee] to archive 2024-11-08T09:22:57,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:22:57,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a1b991f93d374902b6afc53e611f52f5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a1b991f93d374902b6afc53e611f52f5 2024-11-08T09:22:57,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/607ffacc1d704c75813a30376a6f268e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/607ffacc1d704c75813a30376a6f268e 2024-11-08T09:22:57,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/153c276385c8439686a59d44c0d221fb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/153c276385c8439686a59d44c0d221fb 2024-11-08T09:22:57,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/2ce2a7fcd01b41d696d646157d55b288 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/2ce2a7fcd01b41d696d646157d55b288 2024-11-08T09:22:57,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/882d3ed4dd124538ae861a304f12c85c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/882d3ed4dd124538ae861a304f12c85c 2024-11-08T09:22:57,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/61074a6545be46439b919fa4bd4f3775 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/61074a6545be46439b919fa4bd4f3775 2024-11-08T09:22:57,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/c4ab6bda3c9d417ca380bc583602ded5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/c4ab6bda3c9d417ca380bc583602ded5 2024-11-08T09:22:57,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/389517c9303048d787abbe2db386fdd2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/389517c9303048d787abbe2db386fdd2 2024-11-08T09:22:57,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac3d698795564012a0a2253a6b522339 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac3d698795564012a0a2253a6b522339 2024-11-08T09:22:57,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4246eb446abf41e8964cc029cf6d8fd7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4246eb446abf41e8964cc029cf6d8fd7 2024-11-08T09:22:57,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/561fc9ac67644c58b9534ff8e6c36d9a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/561fc9ac67644c58b9534ff8e6c36d9a 2024-11-08T09:22:57,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac59503d178342e99e01c6d6ddfa5b5e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/ac59503d178342e99e01c6d6ddfa5b5e 2024-11-08T09:22:57,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/7ebd68d304184dd8879086c5a5152ec2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/7ebd68d304184dd8879086c5a5152ec2 2024-11-08T09:22:57,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a076ed203726485f9de7d4b8ed0d3863 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a076ed203726485f9de7d4b8ed0d3863 2024-11-08T09:22:57,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/5411d6e072b1456091b0ed7666b1da22 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/5411d6e072b1456091b0ed7666b1da22 2024-11-08T09:22:57,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8033915f52874e778aa1993198d6ee3d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8033915f52874e778aa1993198d6ee3d 2024-11-08T09:22:57,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/39ff885b1c7a430a852aab070c017688 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/39ff885b1c7a430a852aab070c017688 2024-11-08T09:22:57,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a537f39447bc4c68b69ee1963a467589 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a537f39447bc4c68b69ee1963a467589 2024-11-08T09:22:57,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/843def7fa5104b37b5dc60e635867fca to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/843def7fa5104b37b5dc60e635867fca 2024-11-08T09:22:57,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/79193eca0963457da80fb99664063bfa to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/79193eca0963457da80fb99664063bfa 2024-11-08T09:22:57,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/915900c8120c409798f1b9216bf13e4f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/915900c8120c409798f1b9216bf13e4f 2024-11-08T09:22:57,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8d6c14739cd24054ba01f214fde5c8bc to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/8d6c14739cd24054ba01f214fde5c8bc 2024-11-08T09:22:57,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/67677885f0cd48b3b1b71eb075476cd7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/67677885f0cd48b3b1b71eb075476cd7 2024-11-08T09:22:57,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/21889d853d9c41079572ce322e76f3a4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/21889d853d9c41079572ce322e76f3a4 2024-11-08T09:22:57,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a05963225b2c4cdcaafb5aaf8a7d6ad0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/a05963225b2c4cdcaafb5aaf8a7d6ad0 2024-11-08T09:22:57,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/e680eab4dc3748dfbe3d58df2a6312ee to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/e680eab4dc3748dfbe3d58df2a6312ee 2024-11-08T09:22:57,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/77a30783f6e341c6b8d1d89fa0287bab, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/93aa794b1ecf44c7a4d1409f2728cd4c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/9405b6dc0cd94b24803fb4c877358020, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/7224f5a2936547f9b851dee225d05415, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2066359bebac43e89bec3c183bbbfa83, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3614a1381946470f86010a961f8a6489, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eeae8acd337d46caa4519ef3355e4fc3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/348668329095479f9ede52de24f03c8e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/21d6b9cb1ef54d768d8b72fb863d4e84, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/69a5e35c16bd45dcbde1522409660025, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2457e865544e4a5cb1be519d386c5d7e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/55916363ac994d42ae9d2d9cc9c8141b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/56ae784dd5d94ec59919d1cd8e230b4d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5147ce80088f4fd9a0ff0a8624bda41b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a0559add7fe4cb2a84960bc3ca38204, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a5eb66a1dd24f1c918fe072b3ec7b18, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/82c1a21390c14a10b7919adbfd942d66, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/e724f861da734694a4b55fea489bdc31, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/af5104afc0e9421e85c19a6dd77e1d22, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/b7a2b9bcf0ce4590a70bb8ade82537e3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/be7ebc2f813d4baeab522dcbb4d6875f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/45650774b81d4a309f669c2b92b191b3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1693aff22ab94eb18f641e2e97e1b65f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/f2d1e16cac0549458741fa45cdaf632e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/83fd6703c36d485dbc474114c5309282, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3abf891e83394768bd1d909c92154954] to archive 2024-11-08T09:22:57,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:22:57,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/77a30783f6e341c6b8d1d89fa0287bab to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/77a30783f6e341c6b8d1d89fa0287bab 2024-11-08T09:22:57,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/93aa794b1ecf44c7a4d1409f2728cd4c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/93aa794b1ecf44c7a4d1409f2728cd4c 2024-11-08T09:22:57,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/9405b6dc0cd94b24803fb4c877358020 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/9405b6dc0cd94b24803fb4c877358020 2024-11-08T09:22:57,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/7224f5a2936547f9b851dee225d05415 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/7224f5a2936547f9b851dee225d05415 2024-11-08T09:22:57,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2066359bebac43e89bec3c183bbbfa83 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2066359bebac43e89bec3c183bbbfa83 2024-11-08T09:22:57,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3614a1381946470f86010a961f8a6489 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3614a1381946470f86010a961f8a6489 2024-11-08T09:22:57,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eeae8acd337d46caa4519ef3355e4fc3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eeae8acd337d46caa4519ef3355e4fc3 2024-11-08T09:22:57,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/348668329095479f9ede52de24f03c8e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/348668329095479f9ede52de24f03c8e 2024-11-08T09:22:57,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/21d6b9cb1ef54d768d8b72fb863d4e84 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/21d6b9cb1ef54d768d8b72fb863d4e84 2024-11-08T09:22:57,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/69a5e35c16bd45dcbde1522409660025 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/69a5e35c16bd45dcbde1522409660025 2024-11-08T09:22:57,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2457e865544e4a5cb1be519d386c5d7e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/2457e865544e4a5cb1be519d386c5d7e 2024-11-08T09:22:57,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/55916363ac994d42ae9d2d9cc9c8141b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/55916363ac994d42ae9d2d9cc9c8141b 2024-11-08T09:22:57,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/56ae784dd5d94ec59919d1cd8e230b4d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/56ae784dd5d94ec59919d1cd8e230b4d 2024-11-08T09:22:57,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5147ce80088f4fd9a0ff0a8624bda41b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5147ce80088f4fd9a0ff0a8624bda41b 2024-11-08T09:22:57,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a0559add7fe4cb2a84960bc3ca38204 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a0559add7fe4cb2a84960bc3ca38204 2024-11-08T09:22:57,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a5eb66a1dd24f1c918fe072b3ec7b18 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/5a5eb66a1dd24f1c918fe072b3ec7b18 2024-11-08T09:22:57,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/82c1a21390c14a10b7919adbfd942d66 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/82c1a21390c14a10b7919adbfd942d66 2024-11-08T09:22:57,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/e724f861da734694a4b55fea489bdc31 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/e724f861da734694a4b55fea489bdc31 2024-11-08T09:22:57,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/af5104afc0e9421e85c19a6dd77e1d22 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/af5104afc0e9421e85c19a6dd77e1d22 2024-11-08T09:22:57,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/b7a2b9bcf0ce4590a70bb8ade82537e3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/b7a2b9bcf0ce4590a70bb8ade82537e3 2024-11-08T09:22:57,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/be7ebc2f813d4baeab522dcbb4d6875f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/be7ebc2f813d4baeab522dcbb4d6875f 2024-11-08T09:22:57,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/45650774b81d4a309f669c2b92b191b3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/45650774b81d4a309f669c2b92b191b3 2024-11-08T09:22:57,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1693aff22ab94eb18f641e2e97e1b65f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1693aff22ab94eb18f641e2e97e1b65f 2024-11-08T09:22:57,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/f2d1e16cac0549458741fa45cdaf632e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/f2d1e16cac0549458741fa45cdaf632e 2024-11-08T09:22:57,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/83fd6703c36d485dbc474114c5309282 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/83fd6703c36d485dbc474114c5309282 2024-11-08T09:22:57,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3abf891e83394768bd1d909c92154954 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/3abf891e83394768bd1d909c92154954 2024-11-08T09:22:57,104 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/recovered.edits/406.seqid, newMaxSeqId=406, maxSeqId=1 2024-11-08T09:22:57,105 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. 2024-11-08T09:22:57,105 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 6f50fcf560ee3a7bd187b06bd37f77bb: Waiting for close lock at 1731057774812Running coprocessor pre-close hooks at 1731057774812Disabling compacts and flushes for region at 1731057774812Disabling writes for close at 1731057775650 (+838 ms)Obtaining lock to block concurrent updates at 1731057775650Preparing flush snapshotting stores in 6f50fcf560ee3a7bd187b06bd37f77bb at 1731057775650Finished memstore snapshotting TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731057775650Flushing stores of TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb. at 1731057775650Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/A: creating writer at 1731057775651 (+1 ms)Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/A: appending metadata at 1731057775653 (+2 ms)Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/A: closing flushed file at 1731057775654 (+1 ms)Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/B: creating writer at 1731057776063 (+409 ms)Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/B: appending metadata at 1731057776066 (+3 ms)Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/B: closing flushed file at 1731057776066Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/C: creating writer at 1731057776525 (+459 ms)Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/C: appending metadata at 1731057776532 (+7 ms)Flushing 6f50fcf560ee3a7bd187b06bd37f77bb/C: closing flushed file at 1731057776532Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60c09253: reopening flushed file at 1731057776950 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd2d724: reopening flushed file at 1731057776958 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2396fe48: reopening flushed file at 1731057776964 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 6f50fcf560ee3a7bd187b06bd37f77bb in 1320ms, sequenceid=403, compaction requested=true at 1731057776970 (+6 ms)Writing region close event to WAL at 1731057777100 (+130 ms)Running coprocessor post-close hooks at 1731057777104 (+4 ms)Closed at 1731057777105 (+1 ms) 2024-11-08T09:22:57,108 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:57,108 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=6f50fcf560ee3a7bd187b06bd37f77bb, regionState=CLOSED 2024-11-08T09:22:57,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:22:57,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T09:22:57,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 6f50fcf560ee3a7bd187b06bd37f77bb, server=cf4d76213909,39151,1731057746690 in 2.4570 sec 2024-11-08T09:22:57,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-08T09:22:57,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f50fcf560ee3a7bd187b06bd37f77bb, UNASSIGN in 2.4650 sec 2024-11-08T09:22:57,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-08T09:22:57,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4730 sec 2024-11-08T09:22:57,120 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057777120"}]},"ts":"1731057777120"} 2024-11-08T09:22:57,122 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T09:22:57,122 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T09:22:57,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4900 sec 2024-11-08T09:22:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T09:22:58,802 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:58,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T09:22:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:58,813 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:58,815 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T09:22:58,820 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:58,824 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/recovered.edits] 2024-11-08T09:22:58,827 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/742be13d1e1041788f280e54bbd04b1c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/742be13d1e1041788f280e54bbd04b1c 2024-11-08T09:22:58,829 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b87f8da825ba472d9fd10d47a73f737f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/b87f8da825ba472d9fd10d47a73f737f 2024-11-08T09:22:58,830 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d5828fb7b64c4e2ab20ad11d019a9ba8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/A/d5828fb7b64c4e2ab20ad11d019a9ba8 2024-11-08T09:22:58,833 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/388684b52a984a42b3029e4705e8f428 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/388684b52a984a42b3029e4705e8f428 2024-11-08T09:22:58,836 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4e93abddd82d42b3a82047552e7839a1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/4e93abddd82d42b3a82047552e7839a1 2024-11-08T09:22:58,850 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/93fc873ef42348cd8bcf2a8c154d3400 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/B/93fc873ef42348cd8bcf2a8c154d3400 2024-11-08T09:22:58,854 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/021ea7ea204b48379da48f5e6403587b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/021ea7ea204b48379da48f5e6403587b 2024-11-08T09:22:58,855 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1fb852928d8c4152b8887064f2c7021b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/1fb852928d8c4152b8887064f2c7021b 2024-11-08T09:22:58,856 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eafa8df5f6d2491fb1552ef764446606 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/C/eafa8df5f6d2491fb1552ef764446606 2024-11-08T09:22:58,860 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/recovered.edits/406.seqid to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb/recovered.edits/406.seqid 2024-11-08T09:22:58,860 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/6f50fcf560ee3a7bd187b06bd37f77bb 2024-11-08T09:22:58,860 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T09:22:58,865 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:58,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-08T09:22:58,873 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T09:22:58,899 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T09:22:58,900 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:58,900 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T09:22:58,901 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731057778901"}]},"ts":"9223372036854775807"} 2024-11-08T09:22:58,905 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T09:22:58,905 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6f50fcf560ee3a7bd187b06bd37f77bb, NAME => 'TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T09:22:58,906 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T09:22:58,906 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731057778906"}]},"ts":"9223372036854775807"} 2024-11-08T09:22:58,909 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T09:22:58,910 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:58,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-08T09:22:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T09:22:58,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T09:22:58,923 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:22:58,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;cf4d76213909:39151-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_93588470_22 at /127.0.0.1:34118 [Waiting for operation #275] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/cf4d76213909:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2017930777_22 at /127.0.0.1:33018 [Waiting for operation #264] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=528 (was 578), ProcessCount=11 (was 11), AvailableMemoryMB=5668 (was 6587) 2024-11-08T09:22:58,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=528, ProcessCount=11, AvailableMemoryMB=5667 2024-11-08T09:22:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:22:58,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:22:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T09:22:58,948 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T09:22:58,949 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:22:58,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-08T09:22:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T09:22:58,950 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T09:22:58,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741924_1100 (size=959) 2024-11-08T09:22:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T09:22:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T09:22:59,359 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:22:59,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741925_1101 (size=53) 2024-11-08T09:22:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T09:22:59,767 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:22:59,767 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e5919dd324cd2c737f9099915b329cd6, disabling compactions & flushes 2024-11-08T09:22:59,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:22:59,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:22:59,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. after waiting 0 ms 2024-11-08T09:22:59,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:22:59,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:22:59,769 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e5919dd324cd2c737f9099915b329cd6: Waiting for close lock at 1731057779767Disabling compacts and flushes for region at 1731057779767Disabling writes for close at 1731057779768 (+1 ms)Writing region close event to WAL at 1731057779768Closed at 1731057779768 2024-11-08T09:22:59,772 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T09:22:59,773 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731057779772"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731057779772"}]},"ts":"1731057779772"} 2024-11-08T09:22:59,775 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T09:22:59,777 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T09:22:59,777 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057779777"}]},"ts":"1731057779777"} 2024-11-08T09:22:59,779 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T09:22:59,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, ASSIGN}] 2024-11-08T09:22:59,781 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, ASSIGN 2024-11-08T09:22:59,782 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, ASSIGN; state=OFFLINE, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=false 2024-11-08T09:22:59,933 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:22:59,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, ASSIGN because future has completed 2024-11-08T09:22:59,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:23:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T09:23:00,098 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:00,098 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:23:00,099 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,099 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:23:00,099 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,099 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,102 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,105 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5919dd324cd2c737f9099915b329cd6 columnFamilyName A 2024-11-08T09:23:00,106 DEBUG [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:00,107 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(327): Store=e5919dd324cd2c737f9099915b329cd6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:00,107 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,109 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5919dd324cd2c737f9099915b329cd6 columnFamilyName B 2024-11-08T09:23:00,109 DEBUG [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:00,109 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(327): Store=e5919dd324cd2c737f9099915b329cd6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:00,110 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,111 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5919dd324cd2c737f9099915b329cd6 columnFamilyName C 2024-11-08T09:23:00,111 DEBUG [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:00,112 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(327): Store=e5919dd324cd2c737f9099915b329cd6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:00,112 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,113 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,113 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,115 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,115 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,116 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:23:00,117 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,120 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:23:00,121 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened e5919dd324cd2c737f9099915b329cd6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66737474, jitterRate=-0.005534142255783081}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:23:00,121 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:00,122 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for e5919dd324cd2c737f9099915b329cd6: Running coprocessor pre-open hook at 1731057780099Writing region info on filesystem at 1731057780099Initializing all the Stores at 1731057780101 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057780101Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057780101Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057780101Cleaning up temporary data from old regions at 1731057780115 (+14 ms)Running coprocessor post-open hooks at 1731057780121 (+6 ms)Region opened successfully at 1731057780122 (+1 ms) 2024-11-08T09:23:00,123 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., pid=30, masterSystemTime=1731057780093 2024-11-08T09:23:00,126 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:00,126 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:00,126 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:00,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:23:00,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-08T09:23:00,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 in 189 msec 2024-11-08T09:23:00,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T09:23:00,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, ASSIGN in 352 msec 2024-11-08T09:23:00,135 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T09:23:00,135 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057780135"}]},"ts":"1731057780135"} 2024-11-08T09:23:00,137 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T09:23:00,138 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T09:23:00,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-08T09:23:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T09:23:01,094 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:01,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e30321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:01,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:01,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:01,100 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5172ce1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:01,104 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:01,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bdc2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:01,105 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:01,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:01,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:01,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:01,109 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T09:23:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,35807,1731057745600 2024-11-08T09:23:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47569338 2024-11-08T09:23:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T09:23:01,111 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T09:23:01,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:23:01,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:23:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:01,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741926_1102 (size=995) 2024-11-08T09:23:01,536 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T09:23:01,536 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T09:23:01,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:23:01,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, REOPEN/MOVE}] 2024-11-08T09:23:01,601 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, REOPEN/MOVE 2024-11-08T09:23:01,602 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:01,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, REOPEN/MOVE because future has completed 2024-11-08T09:23:01,604 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:23:01,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:23:01,757 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:01,757 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:23:01,757 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing e5919dd324cd2c737f9099915b329cd6, disabling compactions & flushes 2024-11-08T09:23:01,757 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:01,757 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:01,757 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. after waiting 0 ms 2024-11-08T09:23:01,757 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:01,762 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T09:23:01,762 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:01,762 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for e5919dd324cd2c737f9099915b329cd6: Waiting for close lock at 1731057781757Running coprocessor pre-close hooks at 1731057781757Disabling compacts and flushes for region at 1731057781757Disabling writes for close at 1731057781757Writing region close event to WAL at 1731057781758 (+1 ms)Running coprocessor post-close hooks at 1731057781762 (+4 ms)Closed at 1731057781762 2024-11-08T09:23:01,762 WARN [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: e5919dd324cd2c737f9099915b329cd6 to self. 2024-11-08T09:23:01,765 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:01,765 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=CLOSED 2024-11-08T09:23:01,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:23:01,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T09:23:01,772 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, REOPEN/MOVE; state=CLOSED, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=true 2024-11-08T09:23:01,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 in 165 msec 2024-11-08T09:23:01,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:01,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, REOPEN/MOVE because future has completed 2024-11-08T09:23:01,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:23:02,084 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,084 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:23:02,085 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,085 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:23:02,085 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,085 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,092 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,098 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5919dd324cd2c737f9099915b329cd6 columnFamilyName A 2024-11-08T09:23:02,100 DEBUG [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:02,100 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(327): Store=e5919dd324cd2c737f9099915b329cd6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:02,101 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,102 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5919dd324cd2c737f9099915b329cd6 columnFamilyName B 2024-11-08T09:23:02,102 DEBUG [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:02,102 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(327): Store=e5919dd324cd2c737f9099915b329cd6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:02,103 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,103 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5919dd324cd2c737f9099915b329cd6 columnFamilyName C 2024-11-08T09:23:02,103 DEBUG [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:02,104 INFO [StoreOpener-e5919dd324cd2c737f9099915b329cd6-1 {}] regionserver.HStore(327): Store=e5919dd324cd2c737f9099915b329cd6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:02,104 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,105 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,106 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,107 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,107 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,108 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:23:02,109 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,110 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened e5919dd324cd2c737f9099915b329cd6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73801504, jitterRate=0.09972810745239258}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:23:02,110 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,111 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for e5919dd324cd2c737f9099915b329cd6: Running coprocessor pre-open hook at 1731057782085Writing region info on filesystem at 1731057782085Initializing all the Stores at 1731057782087 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057782087Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057782088 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057782088Cleaning up temporary data from old regions at 1731057782107 (+19 ms)Running coprocessor post-open hooks at 1731057782110 (+3 ms)Region opened successfully at 1731057782111 (+1 ms) 2024-11-08T09:23:02,112 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., pid=35, masterSystemTime=1731057782078 2024-11-08T09:23:02,115 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,115 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,115 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=OPEN, openSeqNum=5, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:23:02,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-08T09:23:02,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 in 193 msec 2024-11-08T09:23:02,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-08T09:23:02,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, REOPEN/MOVE in 521 msec 2024-11-08T09:23:02,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T09:23:02,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-08T09:23:02,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0110 sec 2024-11-08T09:23:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T09:23:02,135 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:02,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,138 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79a8b7c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,140 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f321aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,142 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587cade9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,144 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658c82a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5c81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,147 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66484e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8d4b4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749d2404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546683d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,155 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596ede02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,157 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a83e9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686ed360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c86761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a14e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,164 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ecc87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,166 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40682de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17c234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,169 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae94d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,170 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c6f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197f7df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:02,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:02,173 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c3e9ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:02,174 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:02,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:02,177 DEBUG [Thread-539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,177 DEBUG [Thread-556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,177 DEBUG [Thread-554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,177 DEBUG [Thread-545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-08T09:23:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T09:23:02,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,180 DEBUG [Thread-556 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,180 DEBUG [Thread-556 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,181 DEBUG [Thread-550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,181 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:02,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:02,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,182 DEBUG [Thread-552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,192 DEBUG [Thread-543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,192 DEBUG [Thread-541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,192 DEBUG [Thread-547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:02,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:23:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,210 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:02,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:23:02,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081728be38b2cf405ea6b74a64dbec8bc3_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057782206/Put/seqid=0 2024-11-08T09:23:02,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057842256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057842258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057842262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057842264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T09:23:02,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057842273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741927_1103 (size=14594) 2024-11-08T09:23:02,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:02,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081728be38b2cf405ea6b74a64dbec8bc3_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081728be38b2cf405ea6b74a64dbec8bc3_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a18871ebb81d4be4b1f3f4f350e59d87, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:02,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a18871ebb81d4be4b1f3f4f350e59d87 is 175, key is test_row_0/A:col10/1731057782206/Put/seqid=0 2024-11-08T09:23:02,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741928_1104 (size=39549) 2024-11-08T09:23:02,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a18871ebb81d4be4b1f3f4f350e59d87 2024-11-08T09:23:02,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T09:23:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/281ed49c36794c9aada9b00e23933ef6 is 50, key is test_row_0/B:col10/1731057782206/Put/seqid=0 2024-11-08T09:23:02,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741929_1105 (size=12001) 2024-11-08T09:23:02,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/281ed49c36794c9aada9b00e23933ef6 2024-11-08T09:23:02,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057842382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057842383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057842383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057842384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057842392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/e35b0bad595d45ca89097a938ba577b8 is 50, key is test_row_0/C:col10/1731057782206/Put/seqid=0 2024-11-08T09:23:02,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741930_1106 (size=12001) 2024-11-08T09:23:02,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/e35b0bad595d45ca89097a938ba577b8 2024-11-08T09:23:02,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a18871ebb81d4be4b1f3f4f350e59d87 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a18871ebb81d4be4b1f3f4f350e59d87 2024-11-08T09:23:02,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a18871ebb81d4be4b1f3f4f350e59d87, entries=200, sequenceid=17, filesize=38.6 K 2024-11-08T09:23:02,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/281ed49c36794c9aada9b00e23933ef6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/281ed49c36794c9aada9b00e23933ef6 2024-11-08T09:23:02,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/281ed49c36794c9aada9b00e23933ef6, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T09:23:02,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/e35b0bad595d45ca89097a938ba577b8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/e35b0bad595d45ca89097a938ba577b8 2024-11-08T09:23:02,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T09:23:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/e35b0bad595d45ca89097a938ba577b8, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T09:23:02,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e5919dd324cd2c737f9099915b329cd6 in 281ms, sequenceid=17, compaction requested=false 2024-11-08T09:23:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T09:23:02,491 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T09:23:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,595 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:02,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:23:02,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110803f4cf96694949fd9f512d0b7392910b_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057782261/Put/seqid=0 2024-11-08T09:23:02,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057842612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057842613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057842616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057842619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741931_1107 (size=14594) 2024-11-08T09:23:02,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057842620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:02,637 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T09:23:02,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110803f4cf96694949fd9f512d0b7392910b_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110803f4cf96694949fd9f512d0b7392910b_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T09:23:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:02,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ed26417eac8c43f283ab9f901c5e9221, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ed26417eac8c43f283ab9f901c5e9221 is 175, key is test_row_0/A:col10/1731057782261/Put/seqid=0 2024-11-08T09:23:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:02,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741932_1108 (size=39549) 2024-11-08T09:23:02,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ed26417eac8c43f283ab9f901c5e9221 2024-11-08T09:23:02,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/9fb6b1f4568f464a8b65e74d72699edf is 50, key is test_row_0/B:col10/1731057782261/Put/seqid=0 2024-11-08T09:23:02,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741933_1109 (size=12001) 2024-11-08T09:23:02,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/9fb6b1f4568f464a8b65e74d72699edf 2024-11-08T09:23:02,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/672d3029a90c419e864ce51380489936 is 50, key is test_row_0/C:col10/1731057782261/Put/seqid=0 2024-11-08T09:23:02,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741934_1110 (size=12001) 2024-11-08T09:23:02,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/672d3029a90c419e864ce51380489936 2024-11-08T09:23:02,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057842732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057842733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057842733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057842736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ed26417eac8c43f283ab9f901c5e9221 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ed26417eac8c43f283ab9f901c5e9221 2024-11-08T09:23:02,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057842742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ed26417eac8c43f283ab9f901c5e9221, entries=200, sequenceid=42, filesize=38.6 K 2024-11-08T09:23:02,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/9fb6b1f4568f464a8b65e74d72699edf as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9fb6b1f4568f464a8b65e74d72699edf 2024-11-08T09:23:02,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9fb6b1f4568f464a8b65e74d72699edf, entries=150, sequenceid=42, filesize=11.7 K 2024-11-08T09:23:02,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/672d3029a90c419e864ce51380489936 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/672d3029a90c419e864ce51380489936 2024-11-08T09:23:02,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/672d3029a90c419e864ce51380489936, entries=150, sequenceid=42, filesize=11.7 K 2024-11-08T09:23:02,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e5919dd324cd2c737f9099915b329cd6 in 177ms, sequenceid=42, compaction requested=false 2024-11-08T09:23:02,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:02,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T09:23:02,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:02,799 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:23:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T09:23:02,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108aed6d3ed0ee24ef79d2e684c2a82ad71_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057782617/Put/seqid=0 2024-11-08T09:23:02,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741935_1111 (size=12154) 2024-11-08T09:23:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:02,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:02,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057842982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057842985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057842987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057842990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057842990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057843101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057843102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057843102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057843102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057843103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:03,239 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108aed6d3ed0ee24ef79d2e684c2a82ad71_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108aed6d3ed0ee24ef79d2e684c2a82ad71_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:03,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/714c4684843b43f4986f94bb9479da38, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/714c4684843b43f4986f94bb9479da38 is 175, key is test_row_0/A:col10/1731057782617/Put/seqid=0 2024-11-08T09:23:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741936_1112 (size=30955) 2024-11-08T09:23:03,250 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/714c4684843b43f4986f94bb9479da38 2024-11-08T09:23:03,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/9972dabfc62142ba9be220a952cbc210 is 50, key is test_row_0/B:col10/1731057782617/Put/seqid=0 2024-11-08T09:23:03,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741937_1113 (size=12001) 2024-11-08T09:23:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T09:23:03,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057843312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057843313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057843313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057843313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057843314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057843621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057843622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057843622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057843622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057843622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:03,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:03,679 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/9972dabfc62142ba9be220a952cbc210 2024-11-08T09:23:03,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/bbc185884d6b438c8d542cbf943a8482 is 50, key is test_row_0/C:col10/1731057782617/Put/seqid=0 2024-11-08T09:23:03,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741938_1114 (size=12001) 2024-11-08T09:23:03,712 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/bbc185884d6b438c8d542cbf943a8482 2024-11-08T09:23:03,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/714c4684843b43f4986f94bb9479da38 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/714c4684843b43f4986f94bb9479da38 2024-11-08T09:23:03,732 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/714c4684843b43f4986f94bb9479da38, entries=150, sequenceid=53, filesize=30.2 K 2024-11-08T09:23:03,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/9972dabfc62142ba9be220a952cbc210 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9972dabfc62142ba9be220a952cbc210 2024-11-08T09:23:03,742 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9972dabfc62142ba9be220a952cbc210, entries=150, sequenceid=53, filesize=11.7 K 2024-11-08T09:23:03,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/bbc185884d6b438c8d542cbf943a8482 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bbc185884d6b438c8d542cbf943a8482 2024-11-08T09:23:03,759 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bbc185884d6b438c8d542cbf943a8482, entries=150, sequenceid=53, filesize=11.7 K 2024-11-08T09:23:03,760 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e5919dd324cd2c737f9099915b329cd6 in 961ms, sequenceid=53, compaction requested=true 2024-11-08T09:23:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-08T09:23:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-08T09:23:03,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T09:23:03,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5820 sec 2024-11-08T09:23:03,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.5910 sec 2024-11-08T09:23:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:04,137 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:04,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T09:23:04,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108130ed4d93828477f8560fe4376e90261_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057784135/Put/seqid=0 2024-11-08T09:23:04,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057844147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057844153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057844154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057844156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057844156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741939_1115 (size=12154) 2024-11-08T09:23:04,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057844261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057844262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057844271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057844271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057844271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T09:23:04,321 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:04,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-08T09:23:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T09:23:04,324 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:04,324 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:04,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T09:23:04,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057844472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057844472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T09:23:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057844482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057844482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057844483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:04,579 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108130ed4d93828477f8560fe4376e90261_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108130ed4d93828477f8560fe4376e90261_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:04,580 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/4601242aeba842b391f94137d3da88b4, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:04,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/4601242aeba842b391f94137d3da88b4 is 175, key is test_row_0/A:col10/1731057784135/Put/seqid=0 2024-11-08T09:23:04,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741940_1116 (size=30955) 2024-11-08T09:23:04,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T09:23:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T09:23:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057844782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057844782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T09:23:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057844792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057844792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057844792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:04,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T09:23:04,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:04,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T09:23:04,992 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/4601242aeba842b391f94137d3da88b4 2024-11-08T09:23:05,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/b960ef53d4754bcc888087a98181dbb9 is 50, key is test_row_0/B:col10/1731057784135/Put/seqid=0 2024-11-08T09:23:05,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741941_1117 (size=12001) 2024-11-08T09:23:05,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/b960ef53d4754bcc888087a98181dbb9 2024-11-08T09:23:05,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/800ea9b7652a48aeb45c673d2b847a24 is 50, key is test_row_0/C:col10/1731057784135/Put/seqid=0 2024-11-08T09:23:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741942_1118 (size=12001) 2024-11-08T09:23:05,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/800ea9b7652a48aeb45c673d2b847a24 2024-11-08T09:23:05,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/4601242aeba842b391f94137d3da88b4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/4601242aeba842b391f94137d3da88b4 2024-11-08T09:23:05,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/4601242aeba842b391f94137d3da88b4, entries=150, sequenceid=81, filesize=30.2 K 2024-11-08T09:23:05,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/b960ef53d4754bcc888087a98181dbb9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b960ef53d4754bcc888087a98181dbb9 2024-11-08T09:23:05,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b960ef53d4754bcc888087a98181dbb9, entries=150, sequenceid=81, filesize=11.7 K 2024-11-08T09:23:05,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/800ea9b7652a48aeb45c673d2b847a24 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/800ea9b7652a48aeb45c673d2b847a24 2024-11-08T09:23:05,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T09:23:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:05,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/800ea9b7652a48aeb45c673d2b847a24, entries=150, sequenceid=81, filesize=11.7 K 2024-11-08T09:23:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:05,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for e5919dd324cd2c737f9099915b329cd6 in 962ms, sequenceid=81, compaction requested=true 2024-11-08T09:23:05,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:05,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:05,100 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T09:23:05,100 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:05,102 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:05,102 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:05,102 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,102 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/281ed49c36794c9aada9b00e23933ef6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9fb6b1f4568f464a8b65e74d72699edf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9972dabfc62142ba9be220a952cbc210, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b960ef53d4754bcc888087a98181dbb9] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=46.9 K 2024-11-08T09:23:05,103 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:05,103 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:05,103 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,103 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a18871ebb81d4be4b1f3f4f350e59d87, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ed26417eac8c43f283ab9f901c5e9221, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/714c4684843b43f4986f94bb9479da38, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/4601242aeba842b391f94137d3da88b4] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=137.7 K 2024-11-08T09:23:05,103 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,103 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a18871ebb81d4be4b1f3f4f350e59d87, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ed26417eac8c43f283ab9f901c5e9221, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/714c4684843b43f4986f94bb9479da38, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/4601242aeba842b391f94137d3da88b4] 2024-11-08T09:23:05,104 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 281ed49c36794c9aada9b00e23933ef6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731057782187 2024-11-08T09:23:05,105 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18871ebb81d4be4b1f3f4f350e59d87, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731057782187 2024-11-08T09:23:05,105 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb6b1f4568f464a8b65e74d72699edf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731057782259 2024-11-08T09:23:05,106 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed26417eac8c43f283ab9f901c5e9221, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731057782259 2024-11-08T09:23:05,106 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 9972dabfc62142ba9be220a952cbc210, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057782598 2024-11-08T09:23:05,107 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 714c4684843b43f4986f94bb9479da38, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057782598 2024-11-08T09:23:05,107 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b960ef53d4754bcc888087a98181dbb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731057782983 2024-11-08T09:23:05,108 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4601242aeba842b391f94137d3da88b4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731057782983 2024-11-08T09:23:05,121 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:05,122 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/38418e31c5a84c4d9109d2c9000a3546 is 50, key is test_row_0/B:col10/1731057784135/Put/seqid=0 2024-11-08T09:23:05,132 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:05,138 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108d64032c321654f4589f41bb943a4c884_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:05,144 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108d64032c321654f4589f41bb943a4c884_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:05,144 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d64032c321654f4589f41bb943a4c884_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:05,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741943_1119 (size=12139) 2024-11-08T09:23:05,162 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/38418e31c5a84c4d9109d2c9000a3546 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/38418e31c5a84c4d9109d2c9000a3546 2024-11-08T09:23:05,172 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into 38418e31c5a84c4d9109d2c9000a3546(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:05,172 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:05,172 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=12, startTime=1731057785100; duration=0sec 2024-11-08T09:23:05,172 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:05,172 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:05,172 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:05,176 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:05,176 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:05,176 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,176 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/e35b0bad595d45ca89097a938ba577b8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/672d3029a90c419e864ce51380489936, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bbc185884d6b438c8d542cbf943a8482, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/800ea9b7652a48aeb45c673d2b847a24] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=46.9 K 2024-11-08T09:23:05,177 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e35b0bad595d45ca89097a938ba577b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731057782187 2024-11-08T09:23:05,178 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 672d3029a90c419e864ce51380489936, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731057782259 2024-11-08T09:23:05,178 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc185884d6b438c8d542cbf943a8482, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057782598 2024-11-08T09:23:05,179 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 800ea9b7652a48aeb45c673d2b847a24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731057782983 2024-11-08T09:23:05,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741944_1120 (size=4469) 2024-11-08T09:23:05,193 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#100 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:05,194 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/e5e2f9a7ee9f4bc5b2f831389388ef54 is 175, key is test_row_0/A:col10/1731057784135/Put/seqid=0 2024-11-08T09:23:05,211 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:05,211 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/6352186285cb49768130450a4d970e3e is 50, key is test_row_0/C:col10/1731057784135/Put/seqid=0 2024-11-08T09:23:05,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741945_1121 (size=31093) 2024-11-08T09:23:05,223 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/e5e2f9a7ee9f4bc5b2f831389388ef54 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e5e2f9a7ee9f4bc5b2f831389388ef54 2024-11-08T09:23:05,232 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into e5e2f9a7ee9f4bc5b2f831389388ef54(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:05,232 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:05,232 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=12, startTime=1731057785099; duration=0sec 2024-11-08T09:23:05,232 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:05,232 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:05,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741946_1122 (size=12139) 2024-11-08T09:23:05,248 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/6352186285cb49768130450a4d970e3e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6352186285cb49768130450a4d970e3e 2024-11-08T09:23:05,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T09:23:05,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,250 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T09:23:05,260 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into 6352186285cb49768130450a4d970e3e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:05,260 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:05,261 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=12, startTime=1731057785100; duration=0sec 2024-11-08T09:23:05,261 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:05,261 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110882bb75cec2a741689b8e49aa41055ea2_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057784146/Put/seqid=0 2024-11-08T09:23:05,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741947_1123 (size=12154) 2024-11-08T09:23:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:05,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:05,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:05,305 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110882bb75cec2a741689b8e49aa41055ea2_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882bb75cec2a741689b8e49aa41055ea2_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/e7dc2212263b4a7296fb8d48b8386c4d, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/e7dc2212263b4a7296fb8d48b8386c4d is 175, key is test_row_0/A:col10/1731057784146/Put/seqid=0 2024-11-08T09:23:05,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741948_1124 (size=30955) 2024-11-08T09:23:05,333 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/e7dc2212263b4a7296fb8d48b8386c4d 2024-11-08T09:23:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057845331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057845331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057845333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057845335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057845336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/23fff7216093440d970395ee9b5101c2 is 50, key is test_row_0/B:col10/1731057784146/Put/seqid=0 2024-11-08T09:23:05,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741949_1125 (size=12001) 2024-11-08T09:23:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057845441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057845441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057845441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057845452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057845452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T09:23:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057845652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057845652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057845652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057845661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057845661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,771 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/23fff7216093440d970395ee9b5101c2 2024-11-08T09:23:05,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/d97f52ae3d204210852171fabea1fa9b is 50, key is test_row_0/C:col10/1731057784146/Put/seqid=0 2024-11-08T09:23:05,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741950_1126 (size=12001) 2024-11-08T09:23:05,812 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/d97f52ae3d204210852171fabea1fa9b 2024-11-08T09:23:05,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/e7dc2212263b4a7296fb8d48b8386c4d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e7dc2212263b4a7296fb8d48b8386c4d 2024-11-08T09:23:05,834 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e7dc2212263b4a7296fb8d48b8386c4d, entries=150, sequenceid=92, filesize=30.2 K 2024-11-08T09:23:05,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/23fff7216093440d970395ee9b5101c2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/23fff7216093440d970395ee9b5101c2 2024-11-08T09:23:05,844 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/23fff7216093440d970395ee9b5101c2, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T09:23:05,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/d97f52ae3d204210852171fabea1fa9b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d97f52ae3d204210852171fabea1fa9b 2024-11-08T09:23:05,851 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d97f52ae3d204210852171fabea1fa9b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T09:23:05,853 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for e5919dd324cd2c737f9099915b329cd6 in 603ms, sequenceid=92, compaction requested=false 2024-11-08T09:23:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-08T09:23:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-08T09:23:05,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-08T09:23:05,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-08T09:23:05,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.5380 sec 2024-11-08T09:23:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:05,964 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:05,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T09:23:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057845971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057845971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080a88680ee4ba4b4dac2006b4c455ff2d_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057785962/Put/seqid=0 2024-11-08T09:23:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057845973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057845975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057845977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:05,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741951_1127 (size=17034) 2024-11-08T09:23:05,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:05,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080a88680ee4ba4b4dac2006b4c455ff2d_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080a88680ee4ba4b4dac2006b4c455ff2d_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:06,000 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b6bfeacb8e45467486c07fc694db32a7, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:06,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b6bfeacb8e45467486c07fc694db32a7 is 175, key is test_row_0/A:col10/1731057785962/Put/seqid=0 2024-11-08T09:23:06,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741952_1128 (size=48139) 2024-11-08T09:23:06,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057846091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057846091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057846092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057846304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057846304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057846304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b6bfeacb8e45467486c07fc694db32a7 2024-11-08T09:23:06,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f2691d57c74543df8ecdf0e741c35f96 is 50, key is test_row_0/B:col10/1731057785962/Put/seqid=0 2024-11-08T09:23:06,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741953_1129 (size=12001) 2024-11-08T09:23:06,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f2691d57c74543df8ecdf0e741c35f96 2024-11-08T09:23:06,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/9be573efab274d1194e3cdb252d7af2a is 50, key is test_row_0/C:col10/1731057785962/Put/seqid=0 2024-11-08T09:23:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741954_1130 (size=12001) 2024-11-08T09:23:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T09:23:06,471 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:06,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-08T09:23:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T09:23:06,474 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:06,475 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:06,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:06,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057846481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057846492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T09:23:06,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057846612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057846612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057846612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:06,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T09:23:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:06,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:06,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T09:23:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:06,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T09:23:06,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/9be573efab274d1194e3cdb252d7af2a 2024-11-08T09:23:06,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b6bfeacb8e45467486c07fc694db32a7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b6bfeacb8e45467486c07fc694db32a7 2024-11-08T09:23:06,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b6bfeacb8e45467486c07fc694db32a7, entries=250, sequenceid=122, filesize=47.0 K 2024-11-08T09:23:06,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f2691d57c74543df8ecdf0e741c35f96 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f2691d57c74543df8ecdf0e741c35f96 2024-11-08T09:23:06,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f2691d57c74543df8ecdf0e741c35f96, entries=150, sequenceid=122, filesize=11.7 K 2024-11-08T09:23:06,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/9be573efab274d1194e3cdb252d7af2a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9be573efab274d1194e3cdb252d7af2a 2024-11-08T09:23:06,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9be573efab274d1194e3cdb252d7af2a, entries=150, sequenceid=122, filesize=11.7 K 2024-11-08T09:23:06,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for e5919dd324cd2c737f9099915b329cd6 in 932ms, sequenceid=122, compaction requested=true 2024-11-08T09:23:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:06,896 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:06,897 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:06,898 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110187 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:06,898 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:06,898 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,898 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e5e2f9a7ee9f4bc5b2f831389388ef54, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e7dc2212263b4a7296fb8d48b8386c4d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b6bfeacb8e45467486c07fc694db32a7] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=107.6 K 2024-11-08T09:23:06,898 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,898 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e5e2f9a7ee9f4bc5b2f831389388ef54, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e7dc2212263b4a7296fb8d48b8386c4d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b6bfeacb8e45467486c07fc694db32a7] 2024-11-08T09:23:06,899 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:06,899 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:06,899 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,900 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/38418e31c5a84c4d9109d2c9000a3546, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/23fff7216093440d970395ee9b5101c2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f2691d57c74543df8ecdf0e741c35f96] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=35.3 K 2024-11-08T09:23:06,900 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5e2f9a7ee9f4bc5b2f831389388ef54, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731057782983 2024-11-08T09:23:06,900 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7dc2212263b4a7296fb8d48b8386c4d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057784141 2024-11-08T09:23:06,900 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 38418e31c5a84c4d9109d2c9000a3546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731057782983 2024-11-08T09:23:06,901 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6bfeacb8e45467486c07fc694db32a7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731057785327 2024-11-08T09:23:06,901 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fff7216093440d970395ee9b5101c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057784141 2024-11-08T09:23:06,902 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f2691d57c74543df8ecdf0e741c35f96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731057785327 2024-11-08T09:23:06,924 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:06,924 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/ba1f9539534c4bf88419a8860b482470 is 50, key is test_row_0/B:col10/1731057785962/Put/seqid=0 2024-11-08T09:23:06,929 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:06,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T09:23:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,934 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T09:23:06,963 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108cb85eca8493e489f931c39458d454a4c_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:06,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a6844f8003db42f58129d0e034fae2f8_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057785972/Put/seqid=0 2024-11-08T09:23:06,966 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108cb85eca8493e489f931c39458d454a4c_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:06,966 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cb85eca8493e489f931c39458d454a4c_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:06,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741955_1131 (size=12241) 2024-11-08T09:23:06,976 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/ba1f9539534c4bf88419a8860b482470 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/ba1f9539534c4bf88419a8860b482470 2024-11-08T09:23:06,986 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into ba1f9539534c4bf88419a8860b482470(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:06,986 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:06,986 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=13, startTime=1731057786896; duration=0sec 2024-11-08T09:23:06,986 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:06,986 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:06,986 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:06,988 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:06,989 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:06,989 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:06,989 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6352186285cb49768130450a4d970e3e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d97f52ae3d204210852171fabea1fa9b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9be573efab274d1194e3cdb252d7af2a] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=35.3 K 2024-11-08T09:23:06,989 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 6352186285cb49768130450a4d970e3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731057782983 2024-11-08T09:23:06,990 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d97f52ae3d204210852171fabea1fa9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057784141 2024-11-08T09:23:06,991 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be573efab274d1194e3cdb252d7af2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731057785327 2024-11-08T09:23:06,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741957_1133 (size=4469) 2024-11-08T09:23:07,005 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#109 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:07,006 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ce3824b3e5a54ead8c1126fb8433f4ab is 175, key is test_row_0/A:col10/1731057785962/Put/seqid=0 2024-11-08T09:23:07,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741956_1132 (size=12154) 2024-11-08T09:23:07,009 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:07,010 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/b096154991554b088ebb39e4a6510f36 is 50, key is test_row_0/C:col10/1731057785962/Put/seqid=0 2024-11-08T09:23:07,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741958_1134 (size=31195) 2024-11-08T09:23:07,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741959_1135 (size=12241) 2024-11-08T09:23:07,033 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ce3824b3e5a54ead8c1126fb8433f4ab as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ce3824b3e5a54ead8c1126fb8433f4ab 2024-11-08T09:23:07,040 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into ce3824b3e5a54ead8c1126fb8433f4ab(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:07,041 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:07,041 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=13, startTime=1731057786896; duration=0sec 2024-11-08T09:23:07,041 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:07,041 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T09:23:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:07,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:07,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057847165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057847166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057847168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057847271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057847272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057847281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:07,413 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a6844f8003db42f58129d0e034fae2f8_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a6844f8003db42f58129d0e034fae2f8_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:07,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/edb2f446f4504efdad4ece5f00c4d963, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:07,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/edb2f446f4504efdad4ece5f00c4d963 is 175, key is test_row_0/A:col10/1731057785972/Put/seqid=0 2024-11-08T09:23:07,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741960_1136 (size=30955) 2024-11-08T09:23:07,437 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/b096154991554b088ebb39e4a6510f36 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/b096154991554b088ebb39e4a6510f36 2024-11-08T09:23:07,450 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into b096154991554b088ebb39e4a6510f36(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:07,451 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:07,451 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=13, startTime=1731057786897; duration=0sec 2024-11-08T09:23:07,451 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:07,451 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:07,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057847482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057847482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057847491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057847501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057847501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,504 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:07,504 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,590 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T09:23:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T09:23:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057847792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057847792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057847801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:07,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:07,819 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/edb2f446f4504efdad4ece5f00c4d963 2024-11-08T09:23:07,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/5d79f3f69df6460d9cc8c70bfecd422d is 50, key is test_row_0/B:col10/1731057785972/Put/seqid=0 2024-11-08T09:23:07,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741961_1137 (size=12001) 2024-11-08T09:23:07,832 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/5d79f3f69df6460d9cc8c70bfecd422d 2024-11-08T09:23:07,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/9e0003b61ba44e0399edb88b82b7fb55 is 50, key is test_row_0/C:col10/1731057785972/Put/seqid=0 2024-11-08T09:23:07,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741962_1138 (size=12001) 2024-11-08T09:23:08,260 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/9e0003b61ba44e0399edb88b82b7fb55 2024-11-08T09:23:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/edb2f446f4504efdad4ece5f00c4d963 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/edb2f446f4504efdad4ece5f00c4d963 2024-11-08T09:23:08,270 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/edb2f446f4504efdad4ece5f00c4d963, entries=150, sequenceid=129, filesize=30.2 K 2024-11-08T09:23:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/5d79f3f69df6460d9cc8c70bfecd422d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/5d79f3f69df6460d9cc8c70bfecd422d 2024-11-08T09:23:08,275 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/5d79f3f69df6460d9cc8c70bfecd422d, entries=150, sequenceid=129, filesize=11.7 K 2024-11-08T09:23:08,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/9e0003b61ba44e0399edb88b82b7fb55 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9e0003b61ba44e0399edb88b82b7fb55 2024-11-08T09:23:08,281 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9e0003b61ba44e0399edb88b82b7fb55, entries=150, sequenceid=129, filesize=11.7 K 2024-11-08T09:23:08,282 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for e5919dd324cd2c737f9099915b329cd6 in 1348ms, sequenceid=129, compaction requested=false 2024-11-08T09:23:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-08T09:23:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-08T09:23:08,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T09:23:08,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8080 sec 2024-11-08T09:23:08,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.8140 sec 2024-11-08T09:23:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:08,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-08T09:23:08,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110881aa03917628408fa95d107f9fc6ba21_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057787166/Put/seqid=0 2024-11-08T09:23:08,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741963_1139 (size=14794) 2024-11-08T09:23:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057848345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057848345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057848345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057848461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057848462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057848462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T09:23:08,621 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:08,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-08T09:23:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T09:23:08,624 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:08,624 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:08,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:08,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057848671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057848672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057848672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,717 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:08,722 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110881aa03917628408fa95d107f9fc6ba21_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110881aa03917628408fa95d107f9fc6ba21_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:08,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f7141d07e99749f6a356f03976fafc6e, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:08,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f7141d07e99749f6a356f03976fafc6e is 175, key is test_row_0/A:col10/1731057787166/Put/seqid=0 2024-11-08T09:23:08,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741964_1140 (size=39749) 2024-11-08T09:23:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T09:23:08,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:08,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:08,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:08,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T09:23:08,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057848981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057848982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057848982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:09,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,130 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f7141d07e99749f6a356f03976fafc6e 2024-11-08T09:23:09,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/b0a4038b40194c758f038b1ed748697d is 50, key is test_row_0/B:col10/1731057787166/Put/seqid=0 2024-11-08T09:23:09,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741965_1141 (size=12151) 2024-11-08T09:23:09,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:09,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:09,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T09:23:09,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:09,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:09,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:09,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057849492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057849492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:09,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057849492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:09,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057849521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:09,524 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:09,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057849531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:09,535 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:09,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/b0a4038b40194c758f038b1ed748697d 2024-11-08T09:23:09,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/4e3fb38e874d4e29b9b807fc389f5460 is 50, key is test_row_0/C:col10/1731057787166/Put/seqid=0 2024-11-08T09:23:09,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741966_1142 (size=12151) 2024-11-08T09:23:09,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:09,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:09,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T09:23:09,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:09,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/4e3fb38e874d4e29b9b807fc389f5460 2024-11-08T09:23:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f7141d07e99749f6a356f03976fafc6e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f7141d07e99749f6a356f03976fafc6e 2024-11-08T09:23:09,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f7141d07e99749f6a356f03976fafc6e, entries=200, sequenceid=163, filesize=38.8 K 2024-11-08T09:23:09,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/b0a4038b40194c758f038b1ed748697d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b0a4038b40194c758f038b1ed748697d 2024-11-08T09:23:09,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b0a4038b40194c758f038b1ed748697d, entries=150, sequenceid=163, filesize=11.9 K 2024-11-08T09:23:09,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/4e3fb38e874d4e29b9b807fc389f5460 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4e3fb38e874d4e29b9b807fc389f5460 2024-11-08T09:23:09,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4e3fb38e874d4e29b9b807fc389f5460, entries=150, sequenceid=163, filesize=11.9 K 2024-11-08T09:23:09,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=26.84 KB/27480 for e5919dd324cd2c737f9099915b329cd6 in 1677ms, sequenceid=163, compaction requested=true 2024-11-08T09:23:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:09,980 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:09,980 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:09,981 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:09,981 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:09,981 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:09,997 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:09,997 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,997 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,997 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/ba1f9539534c4bf88419a8860b482470, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/5d79f3f69df6460d9cc8c70bfecd422d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b0a4038b40194c758f038b1ed748697d] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=35.5 K 2024-11-08T09:23:09,997 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ce3824b3e5a54ead8c1126fb8433f4ab, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/edb2f446f4504efdad4ece5f00c4d963, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f7141d07e99749f6a356f03976fafc6e] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=99.5 K 2024-11-08T09:23:09,997 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:09,997 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ce3824b3e5a54ead8c1126fb8433f4ab, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/edb2f446f4504efdad4ece5f00c4d963, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f7141d07e99749f6a356f03976fafc6e] 2024-11-08T09:23:09,999 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting ba1f9539534c4bf88419a8860b482470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731057785327 2024-11-08T09:23:10,000 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce3824b3e5a54ead8c1126fb8433f4ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731057785327 2024-11-08T09:23:10,000 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d79f3f69df6460d9cc8c70bfecd422d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731057785966 2024-11-08T09:23:10,000 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting edb2f446f4504efdad4ece5f00c4d963, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731057785966 2024-11-08T09:23:10,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T09:23:10,002 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7141d07e99749f6a356f03976fafc6e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731057787163 2024-11-08T09:23:10,002 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a4038b40194c758f038b1ed748697d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731057787166 2024-11-08T09:23:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:10,002 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T09:23:10,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108191192fda2f545a78a631325ca7a386b_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057788305/Put/seqid=0 2024-11-08T09:23:10,014 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:10,015 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/43979c3918ca45bcb7c012f628a71d13 is 50, key is test_row_0/B:col10/1731057787166/Put/seqid=0 2024-11-08T09:23:10,022 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:10,025 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110814911481bc16481697a8e7c8c4f68b5f_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:10,028 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110814911481bc16481697a8e7c8c4f68b5f_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:10,029 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110814911481bc16481697a8e7c8c4f68b5f_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:10,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741968_1144 (size=12493) 2024-11-08T09:23:10,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741967_1143 (size=12304) 2024-11-08T09:23:10,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:10,044 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108191192fda2f545a78a631325ca7a386b_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108191192fda2f545a78a631325ca7a386b_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/545faa17bebf420b833193c9868fc63c, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:10,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/545faa17bebf420b833193c9868fc63c is 175, key is test_row_0/A:col10/1731057788305/Put/seqid=0 2024-11-08T09:23:10,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741969_1145 (size=4469) 2024-11-08T09:23:10,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741970_1146 (size=31105) 2024-11-08T09:23:10,065 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#119 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:10,066 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/2e505c1fb95c439e868685105be44a3b is 175, key is test_row_0/A:col10/1731057787166/Put/seqid=0 2024-11-08T09:23:10,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741971_1147 (size=31447) 2024-11-08T09:23:10,082 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/2e505c1fb95c439e868685105be44a3b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/2e505c1fb95c439e868685105be44a3b 2024-11-08T09:23:10,089 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into 2e505c1fb95c439e868685105be44a3b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:10,089 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:10,089 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=13, startTime=1731057789980; duration=0sec 2024-11-08T09:23:10,089 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:10,089 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:10,089 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:10,092 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:10,092 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:10,092 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:10,092 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/b096154991554b088ebb39e4a6510f36, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9e0003b61ba44e0399edb88b82b7fb55, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4e3fb38e874d4e29b9b807fc389f5460] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=35.5 K 2024-11-08T09:23:10,093 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting b096154991554b088ebb39e4a6510f36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731057785327 2024-11-08T09:23:10,093 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0003b61ba44e0399edb88b82b7fb55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731057785966 2024-11-08T09:23:10,094 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e3fb38e874d4e29b9b807fc389f5460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731057787166 2024-11-08T09:23:10,107 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:10,108 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/bcf1a8a7a5fe4bd38c206ce9e1f2e78f is 50, key is test_row_0/C:col10/1731057787166/Put/seqid=0 2024-11-08T09:23:10,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741972_1148 (size=12493) 2024-11-08T09:23:10,136 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/bcf1a8a7a5fe4bd38c206ce9e1f2e78f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bcf1a8a7a5fe4bd38c206ce9e1f2e78f 2024-11-08T09:23:10,142 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into bcf1a8a7a5fe4bd38c206ce9e1f2e78f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:10,143 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:10,143 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=13, startTime=1731057789980; duration=0sec 2024-11-08T09:23:10,143 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:10,143 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:10,444 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/43979c3918ca45bcb7c012f628a71d13 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/43979c3918ca45bcb7c012f628a71d13 2024-11-08T09:23:10,452 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into 43979c3918ca45bcb7c012f628a71d13(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:10,452 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:10,452 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=13, startTime=1731057789980; duration=0sec 2024-11-08T09:23:10,452 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:10,452 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:10,457 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/545faa17bebf420b833193c9868fc63c 2024-11-08T09:23:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/18a086d909c4464ab3e89ba80801e85f is 50, key is test_row_0/B:col10/1731057788305/Put/seqid=0 2024-11-08T09:23:10,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741973_1149 (size=12151) 2024-11-08T09:23:10,485 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/18a086d909c4464ab3e89ba80801e85f 2024-11-08T09:23:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/06adaab781e6427a8cfe31e10b20356e is 50, key is test_row_0/C:col10/1731057788305/Put/seqid=0 2024-11-08T09:23:10,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741974_1150 (size=12151) 2024-11-08T09:23:10,512 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/06adaab781e6427a8cfe31e10b20356e 2024-11-08T09:23:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/545faa17bebf420b833193c9868fc63c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/545faa17bebf420b833193c9868fc63c 2024-11-08T09:23:10,523 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/545faa17bebf420b833193c9868fc63c, entries=150, sequenceid=170, filesize=30.4 K 2024-11-08T09:23:10,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/18a086d909c4464ab3e89ba80801e85f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/18a086d909c4464ab3e89ba80801e85f 2024-11-08T09:23:10,529 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/18a086d909c4464ab3e89ba80801e85f, entries=150, sequenceid=170, filesize=11.9 K 2024-11-08T09:23:10,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/06adaab781e6427a8cfe31e10b20356e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/06adaab781e6427a8cfe31e10b20356e 2024-11-08T09:23:10,534 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/06adaab781e6427a8cfe31e10b20356e, entries=150, sequenceid=170, filesize=11.9 K 2024-11-08T09:23:10,553 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=6.71 KB/6870 for e5919dd324cd2c737f9099915b329cd6 in 551ms, sequenceid=170, compaction requested=false 2024-11-08T09:23:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-08T09:23:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-08T09:23:10,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T09:23:10,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9310 sec 2024-11-08T09:23:10,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9360 sec 2024-11-08T09:23:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:10,565 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:10,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:23:10,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085d5b0639d9cb4d2d814947e565e1c20f_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057790560/Put/seqid=0 2024-11-08T09:23:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741975_1151 (size=14794) 2024-11-08T09:23:10,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057850607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057850608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057850612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057850721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057850721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057850721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T09:23:10,772 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:10,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-08T09:23:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T09:23:10,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:10,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:10,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:10,826 INFO [master/cf4d76213909:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T09:23:10,826 INFO [master/cf4d76213909:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T09:23:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T09:23:10,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:10,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:10,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057850931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057850931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057850932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:10,992 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:10,997 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085d5b0639d9cb4d2d814947e565e1c20f_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085d5b0639d9cb4d2d814947e565e1c20f_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:10,998 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/344dca6af9ca4aa39a4244655f03a9a8, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:10,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/344dca6af9ca4aa39a4244655f03a9a8 is 175, key is test_row_0/A:col10/1731057790560/Put/seqid=0 2024-11-08T09:23:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741976_1152 (size=39749) 2024-11-08T09:23:11,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T09:23:11,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:11,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057851242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057851252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057851252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:11,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:11,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:11,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T09:23:11,407 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/344dca6af9ca4aa39a4244655f03a9a8 2024-11-08T09:23:11,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f0c0f3892a8d4bcb8c37c36f230fd193 is 50, key is test_row_0/B:col10/1731057790560/Put/seqid=0 2024-11-08T09:23:11,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741977_1153 (size=12151) 2024-11-08T09:23:11,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:11,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:11,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:11,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:11,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057851752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:11,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057851761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:11,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057851771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:11,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f0c0f3892a8d4bcb8c37c36f230fd193 2024-11-08T09:23:11,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/1da4f5508911462eb7d9cd55eca4d249 is 50, key is test_row_0/C:col10/1731057790560/Put/seqid=0 2024-11-08T09:23:11,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741978_1154 (size=12151) 2024-11-08T09:23:11,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:11,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T09:23:12,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:12,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:12,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/1da4f5508911462eb7d9cd55eca4d249 2024-11-08T09:23:12,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/344dca6af9ca4aa39a4244655f03a9a8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/344dca6af9ca4aa39a4244655f03a9a8 2024-11-08T09:23:12,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/344dca6af9ca4aa39a4244655f03a9a8, entries=200, sequenceid=185, filesize=38.8 K 2024-11-08T09:23:12,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f0c0f3892a8d4bcb8c37c36f230fd193 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f0c0f3892a8d4bcb8c37c36f230fd193 2024-11-08T09:23:12,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f0c0f3892a8d4bcb8c37c36f230fd193, entries=150, sequenceid=185, filesize=11.9 K 2024-11-08T09:23:12,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/1da4f5508911462eb7d9cd55eca4d249 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1da4f5508911462eb7d9cd55eca4d249 2024-11-08T09:23:12,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1da4f5508911462eb7d9cd55eca4d249, entries=150, sequenceid=185, filesize=11.9 K 2024-11-08T09:23:12,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e5919dd324cd2c737f9099915b329cd6 in 1713ms, sequenceid=185, compaction requested=true 2024-11-08T09:23:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:12,279 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:12,279 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:12,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:12,280 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:12,280 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:12,281 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:12,281 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:12,281 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,281 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,281 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/2e505c1fb95c439e868685105be44a3b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/545faa17bebf420b833193c9868fc63c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/344dca6af9ca4aa39a4244655f03a9a8] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=99.9 K 2024-11-08T09:23:12,281 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/43979c3918ca45bcb7c012f628a71d13, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/18a086d909c4464ab3e89ba80801e85f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f0c0f3892a8d4bcb8c37c36f230fd193] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=35.9 K 2024-11-08T09:23:12,281 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,281 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/2e505c1fb95c439e868685105be44a3b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/545faa17bebf420b833193c9868fc63c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/344dca6af9ca4aa39a4244655f03a9a8] 2024-11-08T09:23:12,281 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 43979c3918ca45bcb7c012f628a71d13, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731057787166 2024-11-08T09:23:12,282 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e505c1fb95c439e868685105be44a3b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731057787166 2024-11-08T09:23:12,282 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a086d909c4464ab3e89ba80801e85f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731057788305 2024-11-08T09:23:12,282 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 545faa17bebf420b833193c9868fc63c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731057788305 2024-11-08T09:23:12,282 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 344dca6af9ca4aa39a4244655f03a9a8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731057790518 2024-11-08T09:23:12,282 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c0f3892a8d4bcb8c37c36f230fd193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731057790518 2024-11-08T09:23:12,294 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:12,298 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:12,299 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/396f2129ca284fc6b8da7328ae68e770 is 50, key is test_row_0/B:col10/1731057790560/Put/seqid=0 2024-11-08T09:23:12,306 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110839783df1653045b9b3d4202c34f512cb_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:12,308 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110839783df1653045b9b3d4202c34f512cb_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:12,309 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110839783df1653045b9b3d4202c34f512cb_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741979_1155 (size=12595) 2024-11-08T09:23:12,330 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/396f2129ca284fc6b8da7328ae68e770 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/396f2129ca284fc6b8da7328ae68e770 2024-11-08T09:23:12,339 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into 396f2129ca284fc6b8da7328ae68e770(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:12,339 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:12,339 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=13, startTime=1731057792279; duration=0sec 2024-11-08T09:23:12,339 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:12,339 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:12,340 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:12,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741980_1156 (size=4469) 2024-11-08T09:23:12,342 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:12,343 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:12,343 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#126 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:12,343 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,343 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bcf1a8a7a5fe4bd38c206ce9e1f2e78f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/06adaab781e6427a8cfe31e10b20356e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1da4f5508911462eb7d9cd55eca4d249] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=35.9 K 2024-11-08T09:23:12,343 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/86bbb8ab64474c0db6783b2754b2c705 is 175, key is test_row_0/A:col10/1731057790560/Put/seqid=0 2024-11-08T09:23:12,345 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf1a8a7a5fe4bd38c206ce9e1f2e78f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731057787166 2024-11-08T09:23:12,345 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 06adaab781e6427a8cfe31e10b20356e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731057788305 2024-11-08T09:23:12,346 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 1da4f5508911462eb7d9cd55eca4d249, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731057790518 2024-11-08T09:23:12,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T09:23:12,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:12,351 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:23:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741981_1157 (size=31549) 2024-11-08T09:23:12,358 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:12,359 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/8eba501ef35349b789f715ac04d57ef1 is 50, key is test_row_0/C:col10/1731057790560/Put/seqid=0 2024-11-08T09:23:12,365 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/86bbb8ab64474c0db6783b2754b2c705 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/86bbb8ab64474c0db6783b2754b2c705 2024-11-08T09:23:12,375 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into 86bbb8ab64474c0db6783b2754b2c705(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:12,375 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:12,375 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=13, startTime=1731057792279; duration=0sec 2024-11-08T09:23:12,375 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:12,375 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108982a8bf11d9249deb6fe0a9125e22865_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057790591/Put/seqid=0 2024-11-08T09:23:12,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741982_1158 (size=12595) 2024-11-08T09:23:12,403 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/8eba501ef35349b789f715ac04d57ef1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/8eba501ef35349b789f715ac04d57ef1 2024-11-08T09:23:12,418 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into 8eba501ef35349b789f715ac04d57ef1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:12,418 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:12,418 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=13, startTime=1731057792279; duration=0sec 2024-11-08T09:23:12,419 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:12,419 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:12,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741983_1159 (size=12304) 2024-11-08T09:23:12,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:12,436 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108982a8bf11d9249deb6fe0a9125e22865_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108982a8bf11d9249deb6fe0a9125e22865_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:12,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/5fba6a4eea304f7c9d09ce0da2b85908, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/5fba6a4eea304f7c9d09ce0da2b85908 is 175, key is test_row_0/A:col10/1731057790591/Put/seqid=0 2024-11-08T09:23:12,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741984_1160 (size=31105) 2024-11-08T09:23:12,454 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/5fba6a4eea304f7c9d09ce0da2b85908 2024-11-08T09:23:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/faa27142946441b4a523e3b49ffec008 is 50, key is test_row_0/B:col10/1731057790591/Put/seqid=0 2024-11-08T09:23:12,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741985_1161 (size=12151) 2024-11-08T09:23:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057852784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057852786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:12,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057852791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:12,792 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:12,871 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/faa27142946441b4a523e3b49ffec008 2024-11-08T09:23:12,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/ee747d5c505542318e1f008635eb8b31 is 50, key is test_row_0/C:col10/1731057790591/Put/seqid=0 2024-11-08T09:23:12,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741986_1162 (size=12151) 2024-11-08T09:23:12,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057852891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057852891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T09:23:13,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057853101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057853101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,292 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/ee747d5c505542318e1f008635eb8b31 2024-11-08T09:23:13,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/5fba6a4eea304f7c9d09ce0da2b85908 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/5fba6a4eea304f7c9d09ce0da2b85908 2024-11-08T09:23:13,301 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/5fba6a4eea304f7c9d09ce0da2b85908, entries=150, sequenceid=210, filesize=30.4 K 2024-11-08T09:23:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/faa27142946441b4a523e3b49ffec008 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/faa27142946441b4a523e3b49ffec008 2024-11-08T09:23:13,308 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/faa27142946441b4a523e3b49ffec008, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T09:23:13,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/ee747d5c505542318e1f008635eb8b31 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ee747d5c505542318e1f008635eb8b31 2024-11-08T09:23:13,318 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ee747d5c505542318e1f008635eb8b31, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T09:23:13,319 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e5919dd324cd2c737f9099915b329cd6 in 968ms, sequenceid=210, compaction requested=false 2024-11-08T09:23:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-08T09:23:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-08T09:23:13,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-08T09:23:13,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5460 sec 2024-11-08T09:23:13,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.5520 sec 2024-11-08T09:23:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:13,413 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:13,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T09:23:13,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b95ac4138e7a451f9532afa4e9084b28_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057792785/Put/seqid=0 2024-11-08T09:23:13,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741987_1163 (size=17284) 2024-11-08T09:23:13,429 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:13,434 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b95ac4138e7a451f9532afa4e9084b28_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b95ac4138e7a451f9532afa4e9084b28_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:13,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/04328f3ac69244d192190368f9e522a9, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:13,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/04328f3ac69244d192190368f9e522a9 is 175, key is test_row_0/A:col10/1731057792785/Put/seqid=0 2024-11-08T09:23:13,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741988_1164 (size=48389) 2024-11-08T09:23:13,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/04328f3ac69244d192190368f9e522a9 2024-11-08T09:23:13,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/acf2003d41324c20a66cd592882156ed is 50, key is test_row_0/B:col10/1731057792785/Put/seqid=0 2024-11-08T09:23:13,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057853465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741989_1165 (size=12151) 2024-11-08T09:23:13,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/acf2003d41324c20a66cd592882156ed 2024-11-08T09:23:13,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057853468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/4647eb3445a74ce1be742a85eb3afb41 is 50, key is test_row_0/C:col10/1731057792785/Put/seqid=0 2024-11-08T09:23:13,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741990_1166 (size=12151) 2024-11-08T09:23:13,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/4647eb3445a74ce1be742a85eb3afb41 2024-11-08T09:23:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/04328f3ac69244d192190368f9e522a9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/04328f3ac69244d192190368f9e522a9 2024-11-08T09:23:13,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/04328f3ac69244d192190368f9e522a9, entries=250, sequenceid=226, filesize=47.3 K 2024-11-08T09:23:13,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/acf2003d41324c20a66cd592882156ed as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/acf2003d41324c20a66cd592882156ed 2024-11-08T09:23:13,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/acf2003d41324c20a66cd592882156ed, entries=150, sequenceid=226, filesize=11.9 K 2024-11-08T09:23:13,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/4647eb3445a74ce1be742a85eb3afb41 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4647eb3445a74ce1be742a85eb3afb41 2024-11-08T09:23:13,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4647eb3445a74ce1be742a85eb3afb41, entries=150, sequenceid=226, filesize=11.9 K 2024-11-08T09:23:13,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e5919dd324cd2c737f9099915b329cd6 in 104ms, sequenceid=226, compaction requested=true 2024-11-08T09:23:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:13,518 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T09:23:13,518 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:13,520 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:13,520 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:13,520 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:13,520 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:13,520 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/86bbb8ab64474c0db6783b2754b2c705, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/5fba6a4eea304f7c9d09ce0da2b85908, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/04328f3ac69244d192190368f9e522a9] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=108.4 K 2024-11-08T09:23:13,520 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:13,520 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/86bbb8ab64474c0db6783b2754b2c705, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/5fba6a4eea304f7c9d09ce0da2b85908, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/04328f3ac69244d192190368f9e522a9] 2024-11-08T09:23:13,521 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:13,521 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:13,521 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/396f2129ca284fc6b8da7328ae68e770, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/faa27142946441b4a523e3b49ffec008, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/acf2003d41324c20a66cd592882156ed] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=36.0 K 2024-11-08T09:23:13,521 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86bbb8ab64474c0db6783b2754b2c705, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731057790518 2024-11-08T09:23:13,521 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fba6a4eea304f7c9d09ce0da2b85908, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731057790591 2024-11-08T09:23:13,521 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 396f2129ca284fc6b8da7328ae68e770, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731057790518 2024-11-08T09:23:13,522 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting faa27142946441b4a523e3b49ffec008, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731057790591 2024-11-08T09:23:13,523 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting acf2003d41324c20a66cd592882156ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731057792785 2024-11-08T09:23:13,531 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:13,532 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/881e5c7c1c3547eabb5eda8710452d84 is 50, key is test_row_0/B:col10/1731057792785/Put/seqid=0 2024-11-08T09:23:13,532 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04328f3ac69244d192190368f9e522a9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731057792764 2024-11-08T09:23:13,552 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:13,556 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:13,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T09:23:13,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741991_1167 (size=12697) 2024-11-08T09:23:13,567 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108792c05f1876d4fdeaec12af64d81519d_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:13,569 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108792c05f1876d4fdeaec12af64d81519d_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:13,569 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108792c05f1876d4fdeaec12af64d81519d_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:13,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057853583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057853584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c8fcfca8890045ba847b2a3b868a9ca0_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057793466/Put/seqid=0 2024-11-08T09:23:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057853587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057853586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741992_1168 (size=4469) 2024-11-08T09:23:13,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741993_1169 (size=14794) 2024-11-08T09:23:13,604 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#136 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:13,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:13,605 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0d2c923ccb0404aa94ca8321773b6b6 is 175, key is test_row_0/A:col10/1731057792785/Put/seqid=0 2024-11-08T09:23:13,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741994_1170 (size=31651) 2024-11-08T09:23:13,612 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c8fcfca8890045ba847b2a3b868a9ca0_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c8fcfca8890045ba847b2a3b868a9ca0_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:13,614 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ef67451d826d4f2d8665b547bde6870f, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:13,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ef67451d826d4f2d8665b547bde6870f is 175, key is test_row_0/A:col10/1731057793466/Put/seqid=0 2024-11-08T09:23:13,621 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0d2c923ccb0404aa94ca8321773b6b6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0d2c923ccb0404aa94ca8321773b6b6 2024-11-08T09:23:13,630 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into c0d2c923ccb0404aa94ca8321773b6b6(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:13,630 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:13,630 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=13, startTime=1731057793518; duration=0sec 2024-11-08T09:23:13,630 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:13,630 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:13,630 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:13,632 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:13,632 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:13,632 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:13,632 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/8eba501ef35349b789f715ac04d57ef1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ee747d5c505542318e1f008635eb8b31, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4647eb3445a74ce1be742a85eb3afb41] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=36.0 K 2024-11-08T09:23:13,633 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eba501ef35349b789f715ac04d57ef1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731057790518 2024-11-08T09:23:13,633 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee747d5c505542318e1f008635eb8b31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731057790591 2024-11-08T09:23:13,633 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4647eb3445a74ce1be742a85eb3afb41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731057792785 2024-11-08T09:23:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741995_1171 (size=39749) 2024-11-08T09:23:13,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ef67451d826d4f2d8665b547bde6870f 2024-11-08T09:23:13,650 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:13,651 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/c98e9dcc28bf4825b444f529f0301ce0 is 50, key is test_row_0/C:col10/1731057792785/Put/seqid=0 2024-11-08T09:23:13,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/20485a94c39b4e958d684b5f9487c88f is 50, key is test_row_0/B:col10/1731057793466/Put/seqid=0 2024-11-08T09:23:13,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741996_1172 (size=12697) 2024-11-08T09:23:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057853693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,700 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/c98e9dcc28bf4825b444f529f0301ce0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/c98e9dcc28bf4825b444f529f0301ce0 2024-11-08T09:23:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057853701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057853701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057853701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,709 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into c98e9dcc28bf4825b444f529f0301ce0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:13,709 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:13,709 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=13, startTime=1731057793518; duration=0sec 2024-11-08T09:23:13,709 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:13,709 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:13,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741997_1173 (size=12151) 2024-11-08T09:23:13,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/20485a94c39b4e958d684b5f9487c88f 2024-11-08T09:23:13,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/296b8962f1554ab08a7343ef23385e49 is 50, key is test_row_0/C:col10/1731057793466/Put/seqid=0 2024-11-08T09:23:13,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741998_1174 (size=12151) 2024-11-08T09:23:13,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/296b8962f1554ab08a7343ef23385e49 2024-11-08T09:23:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ef67451d826d4f2d8665b547bde6870f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ef67451d826d4f2d8665b547bde6870f 2024-11-08T09:23:13,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ef67451d826d4f2d8665b547bde6870f, entries=200, sequenceid=249, filesize=38.8 K 2024-11-08T09:23:13,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/20485a94c39b4e958d684b5f9487c88f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/20485a94c39b4e958d684b5f9487c88f 2024-11-08T09:23:13,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/20485a94c39b4e958d684b5f9487c88f, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T09:23:13,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/296b8962f1554ab08a7343ef23385e49 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/296b8962f1554ab08a7343ef23385e49 2024-11-08T09:23:13,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/296b8962f1554ab08a7343ef23385e49, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T09:23:13,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e5919dd324cd2c737f9099915b329cd6 in 234ms, sequenceid=249, compaction requested=false 2024-11-08T09:23:13,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:13,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:13,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T09:23:13,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ff20d41021dc4d9d88ed8988bcddbde0_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057793581/Put/seqid=0 2024-11-08T09:23:13,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741999_1175 (size=14994) 2024-11-08T09:23:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057853932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057853933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057853933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057853936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:13,968 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/881e5c7c1c3547eabb5eda8710452d84 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/881e5c7c1c3547eabb5eda8710452d84 2024-11-08T09:23:13,973 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into 881e5c7c1c3547eabb5eda8710452d84(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:13,973 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:13,973 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=13, startTime=1731057793518; duration=0sec 2024-11-08T09:23:13,973 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:13,973 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:14,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057854041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057854041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057854041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057854051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057854251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057854251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057854251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057854261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:14,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ff20d41021dc4d9d88ed8988bcddbde0_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ff20d41021dc4d9d88ed8988bcddbde0_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:14,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/8b69074ea52f45dda9b232ac0d570f4e, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:14,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/8b69074ea52f45dda9b232ac0d570f4e is 175, key is test_row_0/A:col10/1731057793581/Put/seqid=0 2024-11-08T09:23:14,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742000_1176 (size=39949) 2024-11-08T09:23:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057854562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057854562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057854562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057854571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/8b69074ea52f45dda9b232ac0d570f4e 2024-11-08T09:23:14,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/6778e8bb7aa94ec290b21baabe007e92 is 50, key is test_row_0/B:col10/1731057793581/Put/seqid=0 2024-11-08T09:23:14,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742001_1177 (size=12301) 2024-11-08T09:23:14,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057854811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:14,817 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T09:23:14,941 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:14,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-08T09:23:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T09:23:14,944 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:14,944 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:14,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T09:23:15,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057855071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057855071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057855082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057855082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:15,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T09:23:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/6778e8bb7aa94ec290b21baabe007e92 2024-11-08T09:23:15,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/83879346cc2743c4a89e156cb6473af7 is 50, key is test_row_0/C:col10/1731057793581/Put/seqid=0 2024-11-08T09:23:15,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742002_1178 (size=12301) 2024-11-08T09:23:15,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T09:23:15,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:15,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T09:23:15,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T09:23:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T09:23:15,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:15,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/83879346cc2743c4a89e156cb6473af7 2024-11-08T09:23:15,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:15,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/8b69074ea52f45dda9b232ac0d570f4e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/8b69074ea52f45dda9b232ac0d570f4e 2024-11-08T09:23:15,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/8b69074ea52f45dda9b232ac0d570f4e, entries=200, sequenceid=265, filesize=39.0 K 2024-11-08T09:23:15,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/6778e8bb7aa94ec290b21baabe007e92 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/6778e8bb7aa94ec290b21baabe007e92 2024-11-08T09:23:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T09:23:15,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/6778e8bb7aa94ec290b21baabe007e92, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T09:23:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/83879346cc2743c4a89e156cb6473af7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/83879346cc2743c4a89e156cb6473af7 2024-11-08T09:23:15,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/83879346cc2743c4a89e156cb6473af7, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T09:23:15,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e5919dd324cd2c737f9099915b329cd6 in 1679ms, sequenceid=265, compaction requested=true 2024-11-08T09:23:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:15,582 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:15,582 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:15,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:15,583 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:15,583 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:15,584 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,584 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/881e5c7c1c3547eabb5eda8710452d84, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/20485a94c39b4e958d684b5f9487c88f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/6778e8bb7aa94ec290b21baabe007e92] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=36.3 K 2024-11-08T09:23:15,584 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:15,584 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:15,584 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,584 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0d2c923ccb0404aa94ca8321773b6b6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ef67451d826d4f2d8665b547bde6870f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/8b69074ea52f45dda9b232ac0d570f4e] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=108.7 K 2024-11-08T09:23:15,584 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,584 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0d2c923ccb0404aa94ca8321773b6b6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ef67451d826d4f2d8665b547bde6870f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/8b69074ea52f45dda9b232ac0d570f4e] 2024-11-08T09:23:15,585 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 881e5c7c1c3547eabb5eda8710452d84, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731057792785 2024-11-08T09:23:15,585 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d2c923ccb0404aa94ca8321773b6b6, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731057792785 2024-11-08T09:23:15,585 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 20485a94c39b4e958d684b5f9487c88f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731057793451 2024-11-08T09:23:15,585 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef67451d826d4f2d8665b547bde6870f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731057793451 2024-11-08T09:23:15,585 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 6778e8bb7aa94ec290b21baabe007e92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731057793581 2024-11-08T09:23:15,585 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b69074ea52f45dda9b232ac0d570f4e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731057793581 2024-11-08T09:23:15,594 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:15,596 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:15,596 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/8315548c6f914a928dc2fb6d1a1b56b4 is 50, key is test_row_0/B:col10/1731057793581/Put/seqid=0 2024-11-08T09:23:15,609 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108dbdfb4b8f73b46669dec105ec1dfee46_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:15,611 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108dbdfb4b8f73b46669dec105ec1dfee46_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:15,611 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108dbdfb4b8f73b46669dec105ec1dfee46_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:15,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742004_1180 (size=4469) 2024-11-08T09:23:15,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742003_1179 (size=12949) 2024-11-08T09:23:15,639 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/8315548c6f914a928dc2fb6d1a1b56b4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/8315548c6f914a928dc2fb6d1a1b56b4 2024-11-08T09:23:15,648 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into 8315548c6f914a928dc2fb6d1a1b56b4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:15,648 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:15,648 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=13, startTime=1731057795582; duration=0sec 2024-11-08T09:23:15,649 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:15,649 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:15,649 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:15,650 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:15,650 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:15,650 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,650 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/c98e9dcc28bf4825b444f529f0301ce0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/296b8962f1554ab08a7343ef23385e49, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/83879346cc2743c4a89e156cb6473af7] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=36.3 K 2024-11-08T09:23:15,651 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting c98e9dcc28bf4825b444f529f0301ce0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731057792785 2024-11-08T09:23:15,652 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 296b8962f1554ab08a7343ef23385e49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731057793451 2024-11-08T09:23:15,652 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 83879346cc2743c4a89e156cb6473af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731057793581 2024-11-08T09:23:15,662 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:15,663 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/aa66da528aab45e2946b5d408751a31a is 50, key is test_row_0/C:col10/1731057793581/Put/seqid=0 2024-11-08T09:23:15,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742005_1181 (size=12949) 2024-11-08T09:23:15,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T09:23:15,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:15,710 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T09:23:15,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f4d52dd790fd4e11b03bdff0f1f58aa4_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057793935/Put/seqid=0 2024-11-08T09:23:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742006_1182 (size=12454) 2024-11-08T09:23:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:15,732 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f4d52dd790fd4e11b03bdff0f1f58aa4_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f4d52dd790fd4e11b03bdff0f1f58aa4_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:15,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0fbb5c6c92b4872bc5e68d4951a4628, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0fbb5c6c92b4872bc5e68d4951a4628 is 175, key is test_row_0/A:col10/1731057793935/Put/seqid=0 2024-11-08T09:23:15,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742007_1183 (size=31255) 2024-11-08T09:23:16,031 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:16,032 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/0c1da437559d4de682691a7f6fca0181 is 175, key is test_row_0/A:col10/1731057793581/Put/seqid=0 2024-11-08T09:23:16,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742008_1184 (size=31903) 2024-11-08T09:23:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T09:23:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:16,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:16,085 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/aa66da528aab45e2946b5d408751a31a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/aa66da528aab45e2946b5d408751a31a 2024-11-08T09:23:16,091 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into aa66da528aab45e2946b5d408751a31a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:16,091 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:16,091 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=13, startTime=1731057795582; duration=0sec 2024-11-08T09:23:16,092 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:16,092 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:16,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057856101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057856101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,102 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057856102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057856102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,143 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0fbb5c6c92b4872bc5e68d4951a4628 2024-11-08T09:23:16,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/28360282211343eda5f26227742d99e8 is 50, key is test_row_0/B:col10/1731057793935/Put/seqid=0 2024-11-08T09:23:16,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742009_1185 (size=12301) 2024-11-08T09:23:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057856211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057856211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057856211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057856421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057856421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057856421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,441 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/0c1da437559d4de682691a7f6fca0181 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/0c1da437559d4de682691a7f6fca0181 2024-11-08T09:23:16,447 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into 0c1da437559d4de682691a7f6fca0181(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:16,447 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:16,447 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=13, startTime=1731057795582; duration=0sec 2024-11-08T09:23:16,447 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:16,447 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:16,556 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/28360282211343eda5f26227742d99e8 2024-11-08T09:23:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/282f00ec0a9c48cb9149d7f1d1bd2d2d is 50, key is test_row_0/C:col10/1731057793935/Put/seqid=0 2024-11-08T09:23:16,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742010_1186 (size=12301) 2024-11-08T09:23:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057856731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057856731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057856731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:16,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:16,978 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/282f00ec0a9c48cb9149d7f1d1bd2d2d 2024-11-08T09:23:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0fbb5c6c92b4872bc5e68d4951a4628 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0fbb5c6c92b4872bc5e68d4951a4628 2024-11-08T09:23:16,987 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0fbb5c6c92b4872bc5e68d4951a4628, entries=150, sequenceid=289, filesize=30.5 K 2024-11-08T09:23:16,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/28360282211343eda5f26227742d99e8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/28360282211343eda5f26227742d99e8 2024-11-08T09:23:16,992 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/28360282211343eda5f26227742d99e8, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T09:23:16,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/282f00ec0a9c48cb9149d7f1d1bd2d2d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/282f00ec0a9c48cb9149d7f1d1bd2d2d 2024-11-08T09:23:16,997 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/282f00ec0a9c48cb9149d7f1d1bd2d2d, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T09:23:16,998 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for e5919dd324cd2c737f9099915b329cd6 in 1288ms, sequenceid=289, compaction requested=false 2024-11-08T09:23:16,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:16,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:16,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-08T09:23:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-08T09:23:17,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T09:23:17,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0550 sec 2024-11-08T09:23:17,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0590 sec 2024-11-08T09:23:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T09:23:17,091 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:17,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-08T09:23:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T09:23:17,094 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:17,094 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:17,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T09:23:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:17,243 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:17,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T09:23:17,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T09:23:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108202848e884b84f42ac754ff0405eb368_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057796101/Put/seqid=0 2024-11-08T09:23:17,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057857269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057857270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057857271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742011_1187 (size=14994) 2024-11-08T09:23:17,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:17,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108202848e884b84f42ac754ff0405eb368_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108202848e884b84f42ac754ff0405eb368_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:17,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/d3d2e701882b4a4dbc4153739b7d63c7, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:17,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/d3d2e701882b4a4dbc4153739b7d63c7 is 175, key is test_row_0/A:col10/1731057796101/Put/seqid=0 2024-11-08T09:23:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742012_1188 (size=39949) 2024-11-08T09:23:17,296 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/d3d2e701882b4a4dbc4153739b7d63c7 2024-11-08T09:23:17,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/4862a4bea6b343a1a6f8d1650d7e9ff8 is 50, key is test_row_0/B:col10/1731057796101/Put/seqid=0 2024-11-08T09:23:17,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742013_1189 (size=12301) 2024-11-08T09:23:17,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057857381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057857382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057857382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T09:23:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:17,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T09:23:17,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T09:23:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057857591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057857591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057857592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T09:23:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:17,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:17,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/4862a4bea6b343a1a6f8d1650d7e9ff8 2024-11-08T09:23:17,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/a728ee4fb34a485ebfe0c21e3ed0fe03 is 50, key is test_row_0/C:col10/1731057796101/Put/seqid=0 2024-11-08T09:23:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T09:23:17,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742014_1190 (size=12301) 2024-11-08T09:23:17,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/a728ee4fb34a485ebfe0c21e3ed0fe03 2024-11-08T09:23:17,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/d3d2e701882b4a4dbc4153739b7d63c7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/d3d2e701882b4a4dbc4153739b7d63c7 2024-11-08T09:23:17,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/d3d2e701882b4a4dbc4153739b7d63c7, entries=200, sequenceid=308, filesize=39.0 K 2024-11-08T09:23:17,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/4862a4bea6b343a1a6f8d1650d7e9ff8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4862a4bea6b343a1a6f8d1650d7e9ff8 2024-11-08T09:23:17,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4862a4bea6b343a1a6f8d1650d7e9ff8, entries=150, sequenceid=308, filesize=12.0 K 2024-11-08T09:23:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/a728ee4fb34a485ebfe0c21e3ed0fe03 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/a728ee4fb34a485ebfe0c21e3ed0fe03 2024-11-08T09:23:17,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/a728ee4fb34a485ebfe0c21e3ed0fe03, entries=150, sequenceid=308, filesize=12.0 K 2024-11-08T09:23:17,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e5919dd324cd2c737f9099915b329cd6 in 506ms, sequenceid=308, compaction requested=true 2024-11-08T09:23:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:17,749 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:17,749 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:17,750 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:17,750 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:17,750 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:17,750 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:17,750 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,750 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,750 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/8315548c6f914a928dc2fb6d1a1b56b4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/28360282211343eda5f26227742d99e8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4862a4bea6b343a1a6f8d1650d7e9ff8] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=36.7 K 2024-11-08T09:23:17,750 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/0c1da437559d4de682691a7f6fca0181, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0fbb5c6c92b4872bc5e68d4951a4628, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/d3d2e701882b4a4dbc4153739b7d63c7] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=100.7 K 2024-11-08T09:23:17,750 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,750 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/0c1da437559d4de682691a7f6fca0181, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0fbb5c6c92b4872bc5e68d4951a4628, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/d3d2e701882b4a4dbc4153739b7d63c7] 2024-11-08T09:23:17,751 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 8315548c6f914a928dc2fb6d1a1b56b4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731057793581 2024-11-08T09:23:17,751 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c1da437559d4de682691a7f6fca0181, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731057793581 2024-11-08T09:23:17,751 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0fbb5c6c92b4872bc5e68d4951a4628, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057793927 2024-11-08T09:23:17,751 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 28360282211343eda5f26227742d99e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057793927 2024-11-08T09:23:17,751 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d2e701882b4a4dbc4153739b7d63c7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731057796092 2024-11-08T09:23:17,751 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 4862a4bea6b343a1a6f8d1650d7e9ff8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731057796092 2024-11-08T09:23:17,756 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:17,757 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:17,758 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/a43c518b8a244db88a0bba0c0eae8d43 is 50, key is test_row_0/B:col10/1731057796101/Put/seqid=0 2024-11-08T09:23:17,758 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108cf594aafaa7345f9a5066fdc15936672_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:17,760 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108cf594aafaa7345f9a5066fdc15936672_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:17,760 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cf594aafaa7345f9a5066fdc15936672_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:17,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742015_1191 (size=13051) 2024-11-08T09:23:17,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742016_1192 (size=4469) 2024-11-08T09:23:17,767 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#154 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:17,768 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/450ecc48ee724e48800ba4e1e443682c is 175, key is test_row_0/A:col10/1731057796101/Put/seqid=0 2024-11-08T09:23:17,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742017_1193 (size=32005) 2024-11-08T09:23:17,796 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/450ecc48ee724e48800ba4e1e443682c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/450ecc48ee724e48800ba4e1e443682c 2024-11-08T09:23:17,804 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into 450ecc48ee724e48800ba4e1e443682c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:17,804 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:17,804 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=13, startTime=1731057797749; duration=0sec 2024-11-08T09:23:17,804 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:17,804 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:17,805 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:17,806 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:17,806 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:17,806 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,806 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/aa66da528aab45e2946b5d408751a31a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/282f00ec0a9c48cb9149d7f1d1bd2d2d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/a728ee4fb34a485ebfe0c21e3ed0fe03] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=36.7 K 2024-11-08T09:23:17,807 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa66da528aab45e2946b5d408751a31a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731057793581 2024-11-08T09:23:17,807 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 282f00ec0a9c48cb9149d7f1d1bd2d2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057793927 2024-11-08T09:23:17,807 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting a728ee4fb34a485ebfe0c21e3ed0fe03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731057796092 2024-11-08T09:23:17,819 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:17,819 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/f996f24f0b76454f8b7f48ae331a64d9 is 50, key is test_row_0/C:col10/1731057796101/Put/seqid=0 2024-11-08T09:23:17,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T09:23:17,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:17,862 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T09:23:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742018_1194 (size=13051) 2024-11-08T09:23:17,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b99f89a4de0b4405bbc91703279f24a3_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057797269/Put/seqid=0 2024-11-08T09:23:17,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742019_1195 (size=12454) 2024-11-08T09:23:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:17,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:17,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057857937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057857938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057857939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057858052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057858052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057858052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057858122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,124 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,167 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/a43c518b8a244db88a0bba0c0eae8d43 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/a43c518b8a244db88a0bba0c0eae8d43 2024-11-08T09:23:18,172 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into a43c518b8a244db88a0bba0c0eae8d43(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:18,173 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:18,173 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=13, startTime=1731057797749; duration=0sec 2024-11-08T09:23:18,173 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:18,173 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T09:23:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057858262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057858262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057858262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,271 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/f996f24f0b76454f8b7f48ae331a64d9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/f996f24f0b76454f8b7f48ae331a64d9 2024-11-08T09:23:18,276 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into f996f24f0b76454f8b7f48ae331a64d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:18,276 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:18,276 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=13, startTime=1731057797749; duration=0sec 2024-11-08T09:23:18,276 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:18,276 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:18,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:18,307 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b99f89a4de0b4405bbc91703279f24a3_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b99f89a4de0b4405bbc91703279f24a3_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/33e4a690c69142918924a34187e3e106, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/33e4a690c69142918924a34187e3e106 is 175, key is test_row_0/A:col10/1731057797269/Put/seqid=0 2024-11-08T09:23:18,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742020_1196 (size=31255) 2024-11-08T09:23:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057858572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057858572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057858572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:18,714 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/33e4a690c69142918924a34187e3e106 2024-11-08T09:23:18,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/4dedee93b62842cbb287512ccc014e4e is 50, key is test_row_0/B:col10/1731057797269/Put/seqid=0 2024-11-08T09:23:18,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742021_1197 (size=12301) 2024-11-08T09:23:18,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1731057858831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:18,835 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:19,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057859081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:19,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057859081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:19,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057859091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:19,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:19,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:19,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:19,134 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/4dedee93b62842cbb287512ccc014e4e 2024-11-08T09:23:19,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/1e45dbc981b74448b2abbbf3c0c4699d is 50, key is test_row_0/C:col10/1731057797269/Put/seqid=0 2024-11-08T09:23:19,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742022_1198 (size=12301) 2024-11-08T09:23:19,145 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/1e45dbc981b74448b2abbbf3c0c4699d 2024-11-08T09:23:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/33e4a690c69142918924a34187e3e106 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/33e4a690c69142918924a34187e3e106 2024-11-08T09:23:19,157 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/33e4a690c69142918924a34187e3e106, entries=150, sequenceid=328, filesize=30.5 K 2024-11-08T09:23:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/4dedee93b62842cbb287512ccc014e4e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4dedee93b62842cbb287512ccc014e4e 2024-11-08T09:23:19,168 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4dedee93b62842cbb287512ccc014e4e, entries=150, sequenceid=328, filesize=12.0 K 2024-11-08T09:23:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/1e45dbc981b74448b2abbbf3c0c4699d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1e45dbc981b74448b2abbbf3c0c4699d 2024-11-08T09:23:19,175 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1e45dbc981b74448b2abbbf3c0c4699d, entries=150, sequenceid=328, filesize=12.0 K 2024-11-08T09:23:19,176 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for e5919dd324cd2c737f9099915b329cd6 in 1314ms, sequenceid=328, compaction requested=false 2024-11-08T09:23:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-08T09:23:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-08T09:23:19,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T09:23:19,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0830 sec 2024-11-08T09:23:19,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.0880 sec 2024-11-08T09:23:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T09:23:19,242 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:19,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-08T09:23:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T09:23:19,244 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:19,245 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:19,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T09:23:19,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T09:23:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:19,397 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T09:23:19,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108614661f4e57f4538a2caf2d6eb09d094_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057797935/Put/seqid=0 2024-11-08T09:23:19,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742023_1199 (size=12454) 2024-11-08T09:23:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T09:23:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:19,812 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108614661f4e57f4538a2caf2d6eb09d094_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108614661f4e57f4538a2caf2d6eb09d094_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f8f3c344f08b4852b09926d7017089fa, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:19,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f8f3c344f08b4852b09926d7017089fa is 175, key is test_row_0/A:col10/1731057797935/Put/seqid=0 2024-11-08T09:23:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742024_1200 (size=31255) 2024-11-08T09:23:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T09:23:20,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:20,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057860123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057860123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057860123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,229 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f8f3c344f08b4852b09926d7017089fa 2024-11-08T09:23:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/eb4a7f3d291043d58b0d9033036cd28b is 50, key is test_row_0/B:col10/1731057797935/Put/seqid=0 2024-11-08T09:23:20,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057860231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057860231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057860231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742025_1201 (size=12301) 2024-11-08T09:23:20,250 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/eb4a7f3d291043d58b0d9033036cd28b 2024-11-08T09:23:20,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/d01124a4168b47d9a8ca24213b4a0050 is 50, key is test_row_0/C:col10/1731057797935/Put/seqid=0 2024-11-08T09:23:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742026_1202 (size=12301) 2024-11-08T09:23:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T09:23:20,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057860441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057860441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057860441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,661 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/d01124a4168b47d9a8ca24213b4a0050 2024-11-08T09:23:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/f8f3c344f08b4852b09926d7017089fa as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f8f3c344f08b4852b09926d7017089fa 2024-11-08T09:23:20,669 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f8f3c344f08b4852b09926d7017089fa, entries=150, sequenceid=348, filesize=30.5 K 2024-11-08T09:23:20,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/eb4a7f3d291043d58b0d9033036cd28b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/eb4a7f3d291043d58b0d9033036cd28b 2024-11-08T09:23:20,675 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/eb4a7f3d291043d58b0d9033036cd28b, entries=150, sequenceid=348, filesize=12.0 K 2024-11-08T09:23:20,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/d01124a4168b47d9a8ca24213b4a0050 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d01124a4168b47d9a8ca24213b4a0050 2024-11-08T09:23:20,684 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d01124a4168b47d9a8ca24213b4a0050, entries=150, sequenceid=348, filesize=12.0 K 2024-11-08T09:23:20,685 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for e5919dd324cd2c737f9099915b329cd6 in 1288ms, sequenceid=348, compaction requested=true 2024-11-08T09:23:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-08T09:23:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-08T09:23:20,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-08T09:23:20,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4410 sec 2024-11-08T09:23:20,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4460 sec 2024-11-08T09:23:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:20,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:20,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T09:23:20,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110892b598e4feb84c3c83bb26c13e7dd478_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057800754/Put/seqid=0 2024-11-08T09:23:20,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057860774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057860775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742027_1203 (size=12454) 2024-11-08T09:23:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057860776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057860882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057860882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057860891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057861092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057861092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057861101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:21,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110892b598e4feb84c3c83bb26c13e7dd478_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110892b598e4feb84c3c83bb26c13e7dd478_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:21,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ca2d8afe111d45939548675906174758, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:21,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ca2d8afe111d45939548675906174758 is 175, key is test_row_0/A:col10/1731057800754/Put/seqid=0 2024-11-08T09:23:21,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742028_1204 (size=31255) 2024-11-08T09:23:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T09:23:21,392 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:21,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-08T09:23:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T09:23:21,394 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:21,395 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:21,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:21,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057861401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057861401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057861421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T09:23:21,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T09:23:21,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:21,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:21,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:21,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,592 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ca2d8afe111d45939548675906174758 2024-11-08T09:23:21,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/709c82aeb91e432f8101fb6f58a2c30d is 50, key is test_row_0/B:col10/1731057800754/Put/seqid=0 2024-11-08T09:23:21,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742029_1205 (size=12301) 2024-11-08T09:23:21,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T09:23:21,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:21,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:21,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:21,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T09:23:21,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T09:23:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:21,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:21,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52728 deadline: 1731057861911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1731057861911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52688 deadline: 1731057861931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:22,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T09:23:22,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/709c82aeb91e432f8101fb6f58a2c30d 2024-11-08T09:23:22,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:22,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/de6f3ec7842d4b7086827719fa09cbee is 50, key is test_row_0/C:col10/1731057800754/Put/seqid=0 2024-11-08T09:23:22,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742030_1206 (size=12301) 2024-11-08T09:23:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T09:23:22,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T09:23:22,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:22,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52734 deadline: 1731057862161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:22,164 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in e5919dd324cd2c737f9099915b329cd6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:23:22,194 INFO [Thread-556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-556. 2024-11-08T09:23:22,194 DEBUG [Thread-556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:22,194 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,194 DEBUG [Thread-556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:22,195 INFO [Thread-552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-552. 2024-11-08T09:23:22,195 DEBUG [Thread-552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:22,195 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,195 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:22,196 INFO [Thread-550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-550. 2024-11-08T09:23:22,196 DEBUG [Thread-550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:22,196 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,196 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:22,198 INFO [Thread-554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-554. 2024-11-08T09:23:22,198 DEBUG [Thread-554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:22,198 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,198 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:22,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T09:23:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:22,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:22,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/de6f3ec7842d4b7086827719fa09cbee 2024-11-08T09:23:22,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/ca2d8afe111d45939548675906174758 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ca2d8afe111d45939548675906174758 2024-11-08T09:23:22,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ca2d8afe111d45939548675906174758, entries=150, sequenceid=368, filesize=30.5 K 2024-11-08T09:23:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/709c82aeb91e432f8101fb6f58a2c30d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/709c82aeb91e432f8101fb6f58a2c30d 2024-11-08T09:23:22,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/709c82aeb91e432f8101fb6f58a2c30d, entries=150, sequenceid=368, filesize=12.0 K 2024-11-08T09:23:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/de6f3ec7842d4b7086827719fa09cbee as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/de6f3ec7842d4b7086827719fa09cbee 2024-11-08T09:23:22,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/de6f3ec7842d4b7086827719fa09cbee, entries=150, sequenceid=368, filesize=12.0 K 2024-11-08T09:23:22,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e5919dd324cd2c737f9099915b329cd6 in 1692ms, sequenceid=368, compaction requested=true 2024-11-08T09:23:22,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:22,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:22,448 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5919dd324cd2c737f9099915b329cd6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:22,448 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:22,449 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:22,449 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:22,449 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/B is initiating minor compaction (all files) 2024-11-08T09:23:22,449 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/A is initiating minor compaction (all files) 2024-11-08T09:23:22,449 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/B in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,449 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/A in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,449 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/a43c518b8a244db88a0bba0c0eae8d43, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4dedee93b62842cbb287512ccc014e4e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/eb4a7f3d291043d58b0d9033036cd28b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/709c82aeb91e432f8101fb6f58a2c30d] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=48.8 K 2024-11-08T09:23:22,449 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/450ecc48ee724e48800ba4e1e443682c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/33e4a690c69142918924a34187e3e106, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f8f3c344f08b4852b09926d7017089fa, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ca2d8afe111d45939548675906174758] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=122.8 K 2024-11-08T09:23:22,449 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,449 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/450ecc48ee724e48800ba4e1e443682c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/33e4a690c69142918924a34187e3e106, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f8f3c344f08b4852b09926d7017089fa, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ca2d8afe111d45939548675906174758] 2024-11-08T09:23:22,449 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a43c518b8a244db88a0bba0c0eae8d43, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731057796092 2024-11-08T09:23:22,450 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 450ecc48ee724e48800ba4e1e443682c, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731057796092 2024-11-08T09:23:22,450 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dedee93b62842cbb287512ccc014e4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731057797268 2024-11-08T09:23:22,450 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33e4a690c69142918924a34187e3e106, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731057797268 2024-11-08T09:23:22,450 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting eb4a7f3d291043d58b0d9033036cd28b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731057797935 2024-11-08T09:23:22,450 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f3c344f08b4852b09926d7017089fa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731057797935 2024-11-08T09:23:22,451 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 709c82aeb91e432f8101fb6f58a2c30d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731057800121 2024-11-08T09:23:22,451 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca2d8afe111d45939548675906174758, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731057800121 2024-11-08T09:23:22,459 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:22,460 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f8449ea13b6e48d48bbe6a84b01b7e6c is 50, key is test_row_0/B:col10/1731057800754/Put/seqid=0 2024-11-08T09:23:22,463 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:22,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742031_1207 (size=13187) 2024-11-08T09:23:22,466 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108af13b46b6c304d1999005c52034f3ee7_e5919dd324cd2c737f9099915b329cd6 store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:22,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T09:23:22,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,470 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T09:23:22,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ef37487361eb4c309a318c7296269c1f_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057800775/Put/seqid=0 2024-11-08T09:23:22,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742032_1208 (size=12454) 2024-11-08T09:23:22,491 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108af13b46b6c304d1999005c52034f3ee7_e5919dd324cd2c737f9099915b329cd6, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:22,491 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108af13b46b6c304d1999005c52034f3ee7_e5919dd324cd2c737f9099915b329cd6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:22,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742033_1209 (size=4469) 2024-11-08T09:23:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T09:23:22,873 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/f8449ea13b6e48d48bbe6a84b01b7e6c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f8449ea13b6e48d48bbe6a84b01b7e6c 2024-11-08T09:23:22,880 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/B of e5919dd324cd2c737f9099915b329cd6 into f8449ea13b6e48d48bbe6a84b01b7e6c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:22,880 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:22,880 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/B, priority=12, startTime=1731057802448; duration=0sec 2024-11-08T09:23:22,880 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:22,880 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:B 2024-11-08T09:23:22,880 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:22,882 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:22,882 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): e5919dd324cd2c737f9099915b329cd6/C is initiating minor compaction (all files) 2024-11-08T09:23:22,882 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5919dd324cd2c737f9099915b329cd6/C in TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:22,882 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/f996f24f0b76454f8b7f48ae331a64d9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1e45dbc981b74448b2abbbf3c0c4699d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d01124a4168b47d9a8ca24213b4a0050, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/de6f3ec7842d4b7086827719fa09cbee] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp, totalSize=48.8 K 2024-11-08T09:23:22,882 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f996f24f0b76454f8b7f48ae331a64d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731057796092 2024-11-08T09:23:22,883 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e45dbc981b74448b2abbbf3c0c4699d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731057797268 2024-11-08T09:23:22,883 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d01124a4168b47d9a8ca24213b4a0050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731057797935 2024-11-08T09:23:22,883 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting de6f3ec7842d4b7086827719fa09cbee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731057800121 2024-11-08T09:23:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:22,894 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:22,894 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/6db4b98eed524d598ca4893ca934d099 is 50, key is test_row_0/C:col10/1731057800754/Put/seqid=0 2024-11-08T09:23:22,896 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5919dd324cd2c737f9099915b329cd6#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:22,897 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0b7011df09944deb01efa029d11e2aa is 175, key is test_row_0/A:col10/1731057800754/Put/seqid=0 2024-11-08T09:23:22,897 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ef37487361eb4c309a318c7296269c1f_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ef37487361eb4c309a318c7296269c1f_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a4948d074b654fe389277c3e20f8ed46, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:22,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a4948d074b654fe389277c3e20f8ed46 is 175, key is test_row_0/A:col10/1731057800775/Put/seqid=0 2024-11-08T09:23:22,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742034_1210 (size=13187) 2024-11-08T09:23:22,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742036_1212 (size=31255) 2024-11-08T09:23:22,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742035_1211 (size=32141) 2024-11-08T09:23:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:22,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. as already flushing 2024-11-08T09:23:22,923 INFO [Thread-545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-545. 2024-11-08T09:23:22,923 INFO [Thread-547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-547. 2024-11-08T09:23:22,923 DEBUG [Thread-545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:22,923 DEBUG [Thread-547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:22,923 DEBUG [Thread-545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,923 DEBUG [Thread-547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,923 DEBUG [Thread-545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,923 DEBUG [Thread-547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:22,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:22,943 INFO [Thread-539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-539. 2024-11-08T09:23:22,943 DEBUG [Thread-539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:22,943 DEBUG [Thread-539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,943 DEBUG [Thread-539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:22,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:23,305 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a4948d074b654fe389277c3e20f8ed46 2024-11-08T09:23:23,314 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/6db4b98eed524d598ca4893ca934d099 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6db4b98eed524d598ca4893ca934d099 2024-11-08T09:23:23,314 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/c0b7011df09944deb01efa029d11e2aa as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0b7011df09944deb01efa029d11e2aa 2024-11-08T09:23:23,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/7e3374ebcbc44f11a67a4b4fcbf658be is 50, key is test_row_0/B:col10/1731057800775/Put/seqid=0 2024-11-08T09:23:23,319 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/A of e5919dd324cd2c737f9099915b329cd6 into c0b7011df09944deb01efa029d11e2aa(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:23,319 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5919dd324cd2c737f9099915b329cd6/C of e5919dd324cd2c737f9099915b329cd6 into 6db4b98eed524d598ca4893ca934d099(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:23,320 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:23,320 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:23,320 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/C, priority=12, startTime=1731057802448; duration=0sec 2024-11-08T09:23:23,320 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., storeName=e5919dd324cd2c737f9099915b329cd6/A, priority=12, startTime=1731057802447; duration=0sec 2024-11-08T09:23:23,320 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:23,320 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:C 2024-11-08T09:23:23,320 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:23,320 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5919dd324cd2c737f9099915b329cd6:A 2024-11-08T09:23:23,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742037_1213 (size=12301) 2024-11-08T09:23:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T09:23:23,723 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/7e3374ebcbc44f11a67a4b4fcbf658be 2024-11-08T09:23:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/ac31634c539449c087fb12014b236c26 is 50, key is test_row_0/C:col10/1731057800775/Put/seqid=0 2024-11-08T09:23:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742038_1214 (size=12301) 2024-11-08T09:23:24,147 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/ac31634c539449c087fb12014b236c26 2024-11-08T09:23:24,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/a4948d074b654fe389277c3e20f8ed46 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a4948d074b654fe389277c3e20f8ed46 2024-11-08T09:23:24,163 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a4948d074b654fe389277c3e20f8ed46, entries=150, sequenceid=384, filesize=30.5 K 2024-11-08T09:23:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/7e3374ebcbc44f11a67a4b4fcbf658be as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/7e3374ebcbc44f11a67a4b4fcbf658be 2024-11-08T09:23:24,168 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/7e3374ebcbc44f11a67a4b4fcbf658be, entries=150, sequenceid=384, filesize=12.0 K 2024-11-08T09:23:24,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/ac31634c539449c087fb12014b236c26 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ac31634c539449c087fb12014b236c26 2024-11-08T09:23:24,173 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ac31634c539449c087fb12014b236c26, entries=150, sequenceid=384, filesize=12.0 K 2024-11-08T09:23:24,174 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=20.13 KB/20610 for e5919dd324cd2c737f9099915b329cd6 in 1704ms, sequenceid=384, compaction requested=false 2024-11-08T09:23:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e5919dd324cd2c737f9099915b329cd6: 2024-11-08T09:23:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:24,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-08T09:23:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-08T09:23:24,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T09:23:24,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7810 sec 2024-11-08T09:23:24,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.7870 sec 2024-11-08T09:23:24,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T09:23:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T09:23:25,572 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:28,905 INFO [Thread-543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-543. 2024-11-08T09:23:28,906 DEBUG [Thread-543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:28,906 DEBUG [Thread-543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:28,906 DEBUG [Thread-543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:28,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:32,217 INFO [Thread-541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-541. 2024-11-08T09:23:32,217 DEBUG [Thread-541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:32,217 DEBUG [Thread-541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:32,217 DEBUG [Thread-541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:32,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:32,218 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T09:23:32,218 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-08T09:23:32,218 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-08T09:23:32,218 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-08T09:23:32,218 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-08T09:23:32,218 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-08T09:23:32,218 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T09:23:32,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4110 2024-11-08T09:23:32,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4042 2024-11-08T09:23:32,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T09:23:32,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1795 2024-11-08T09:23:32,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5382 rows 2024-11-08T09:23:32,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1795 2024-11-08T09:23:32,219 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5384 rows 2024-11-08T09:23:32,219 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:23:32,219 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:23:32,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:32,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:32,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:32,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T09:23:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T09:23:32,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057812225"}]},"ts":"1731057812225"} 2024-11-08T09:23:32,227 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T09:23:32,227 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T09:23:32,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:23:32,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, UNASSIGN}] 2024-11-08T09:23:32,230 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, UNASSIGN 2024-11-08T09:23:32,231 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:32,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, UNASSIGN because future has completed 2024-11-08T09:23:32,232 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:23:32,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:23:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T09:23:32,386 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:32,386 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:23:32,386 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing e5919dd324cd2c737f9099915b329cd6, disabling compactions & flushes 2024-11-08T09:23:32,386 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:32,386 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:32,386 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. after waiting 0 ms 2024-11-08T09:23:32,386 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:32,386 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing e5919dd324cd2c737f9099915b329cd6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T09:23:32,398 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085b76bdc6a63c4f50b6abc07baf14bd80_e5919dd324cd2c737f9099915b329cd6 is 50, key is test_row_0/A:col10/1731057808902/Put/seqid=0 2024-11-08T09:23:32,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742039_1215 (size=9914) 2024-11-08T09:23:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T09:23:32,804 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:32,812 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085b76bdc6a63c4f50b6abc07baf14bd80_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b76bdc6a63c4f50b6abc07baf14bd80_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:32,813 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b0e38a3a2e95454f9b4e868b4de57e2b, store: [table=TestAcidGuarantees family=A region=e5919dd324cd2c737f9099915b329cd6] 2024-11-08T09:23:32,814 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b0e38a3a2e95454f9b4e868b4de57e2b is 175, key is test_row_0/A:col10/1731057808902/Put/seqid=0 2024-11-08T09:23:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742040_1216 (size=22561) 2024-11-08T09:23:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T09:23:33,219 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b0e38a3a2e95454f9b4e868b4de57e2b 2024-11-08T09:23:33,230 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/00f56302f2d2462aa679fedf0b7d4566 is 50, key is test_row_0/B:col10/1731057808902/Put/seqid=0 2024-11-08T09:23:33,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742041_1217 (size=9857) 2024-11-08T09:23:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T09:23:33,635 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/00f56302f2d2462aa679fedf0b7d4566 2024-11-08T09:23:33,766 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/dd5643b564c94a7a90c232e916e7c452 is 50, key is test_row_0/C:col10/1731057808902/Put/seqid=0 2024-11-08T09:23:33,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742042_1218 (size=9857) 2024-11-08T09:23:34,173 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/dd5643b564c94a7a90c232e916e7c452 2024-11-08T09:23:34,183 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/A/b0e38a3a2e95454f9b4e868b4de57e2b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b0e38a3a2e95454f9b4e868b4de57e2b 2024-11-08T09:23:34,188 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b0e38a3a2e95454f9b4e868b4de57e2b, entries=100, sequenceid=395, filesize=22.0 K 2024-11-08T09:23:34,189 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/B/00f56302f2d2462aa679fedf0b7d4566 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/00f56302f2d2462aa679fedf0b7d4566 2024-11-08T09:23:34,193 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/00f56302f2d2462aa679fedf0b7d4566, entries=100, sequenceid=395, filesize=9.6 K 2024-11-08T09:23:34,193 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/.tmp/C/dd5643b564c94a7a90c232e916e7c452 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/dd5643b564c94a7a90c232e916e7c452 2024-11-08T09:23:34,197 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/dd5643b564c94a7a90c232e916e7c452, entries=100, sequenceid=395, filesize=9.6 K 2024-11-08T09:23:34,198 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e5919dd324cd2c737f9099915b329cd6 in 1811ms, sequenceid=395, compaction requested=true 2024-11-08T09:23:34,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a18871ebb81d4be4b1f3f4f350e59d87, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ed26417eac8c43f283ab9f901c5e9221, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/714c4684843b43f4986f94bb9479da38, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e5e2f9a7ee9f4bc5b2f831389388ef54, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/4601242aeba842b391f94137d3da88b4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e7dc2212263b4a7296fb8d48b8386c4d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b6bfeacb8e45467486c07fc694db32a7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ce3824b3e5a54ead8c1126fb8433f4ab, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/edb2f446f4504efdad4ece5f00c4d963, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f7141d07e99749f6a356f03976fafc6e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/2e505c1fb95c439e868685105be44a3b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/545faa17bebf420b833193c9868fc63c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/344dca6af9ca4aa39a4244655f03a9a8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/86bbb8ab64474c0db6783b2754b2c705, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/5fba6a4eea304f7c9d09ce0da2b85908, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/04328f3ac69244d192190368f9e522a9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0d2c923ccb0404aa94ca8321773b6b6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ef67451d826d4f2d8665b547bde6870f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/8b69074ea52f45dda9b232ac0d570f4e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/0c1da437559d4de682691a7f6fca0181, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0fbb5c6c92b4872bc5e68d4951a4628, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/d3d2e701882b4a4dbc4153739b7d63c7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/450ecc48ee724e48800ba4e1e443682c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/33e4a690c69142918924a34187e3e106, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f8f3c344f08b4852b09926d7017089fa, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ca2d8afe111d45939548675906174758] to archive 2024-11-08T09:23:34,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:23:34,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a18871ebb81d4be4b1f3f4f350e59d87 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a18871ebb81d4be4b1f3f4f350e59d87 2024-11-08T09:23:34,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ed26417eac8c43f283ab9f901c5e9221 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ed26417eac8c43f283ab9f901c5e9221 2024-11-08T09:23:34,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/714c4684843b43f4986f94bb9479da38 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/714c4684843b43f4986f94bb9479da38 2024-11-08T09:23:34,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e5e2f9a7ee9f4bc5b2f831389388ef54 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e5e2f9a7ee9f4bc5b2f831389388ef54 2024-11-08T09:23:34,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/4601242aeba842b391f94137d3da88b4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/4601242aeba842b391f94137d3da88b4 2024-11-08T09:23:34,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e7dc2212263b4a7296fb8d48b8386c4d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/e7dc2212263b4a7296fb8d48b8386c4d 2024-11-08T09:23:34,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b6bfeacb8e45467486c07fc694db32a7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b6bfeacb8e45467486c07fc694db32a7 2024-11-08T09:23:34,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ce3824b3e5a54ead8c1126fb8433f4ab to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ce3824b3e5a54ead8c1126fb8433f4ab 2024-11-08T09:23:34,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/edb2f446f4504efdad4ece5f00c4d963 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/edb2f446f4504efdad4ece5f00c4d963 2024-11-08T09:23:34,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f7141d07e99749f6a356f03976fafc6e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f7141d07e99749f6a356f03976fafc6e 2024-11-08T09:23:34,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/2e505c1fb95c439e868685105be44a3b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/2e505c1fb95c439e868685105be44a3b 2024-11-08T09:23:34,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/545faa17bebf420b833193c9868fc63c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/545faa17bebf420b833193c9868fc63c 2024-11-08T09:23:34,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/344dca6af9ca4aa39a4244655f03a9a8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/344dca6af9ca4aa39a4244655f03a9a8 2024-11-08T09:23:34,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/86bbb8ab64474c0db6783b2754b2c705 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/86bbb8ab64474c0db6783b2754b2c705 2024-11-08T09:23:34,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/5fba6a4eea304f7c9d09ce0da2b85908 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/5fba6a4eea304f7c9d09ce0da2b85908 2024-11-08T09:23:34,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/04328f3ac69244d192190368f9e522a9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/04328f3ac69244d192190368f9e522a9 2024-11-08T09:23:34,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0d2c923ccb0404aa94ca8321773b6b6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0d2c923ccb0404aa94ca8321773b6b6 2024-11-08T09:23:34,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ef67451d826d4f2d8665b547bde6870f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ef67451d826d4f2d8665b547bde6870f 2024-11-08T09:23:34,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/8b69074ea52f45dda9b232ac0d570f4e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/8b69074ea52f45dda9b232ac0d570f4e 2024-11-08T09:23:34,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/0c1da437559d4de682691a7f6fca0181 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/0c1da437559d4de682691a7f6fca0181 2024-11-08T09:23:34,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0fbb5c6c92b4872bc5e68d4951a4628 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0fbb5c6c92b4872bc5e68d4951a4628 2024-11-08T09:23:34,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/d3d2e701882b4a4dbc4153739b7d63c7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/d3d2e701882b4a4dbc4153739b7d63c7 2024-11-08T09:23:34,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/450ecc48ee724e48800ba4e1e443682c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/450ecc48ee724e48800ba4e1e443682c 2024-11-08T09:23:34,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/33e4a690c69142918924a34187e3e106 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/33e4a690c69142918924a34187e3e106 2024-11-08T09:23:34,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f8f3c344f08b4852b09926d7017089fa to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/f8f3c344f08b4852b09926d7017089fa 2024-11-08T09:23:34,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ca2d8afe111d45939548675906174758 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/ca2d8afe111d45939548675906174758 2024-11-08T09:23:34,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/281ed49c36794c9aada9b00e23933ef6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9fb6b1f4568f464a8b65e74d72699edf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9972dabfc62142ba9be220a952cbc210, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/38418e31c5a84c4d9109d2c9000a3546, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b960ef53d4754bcc888087a98181dbb9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/23fff7216093440d970395ee9b5101c2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/ba1f9539534c4bf88419a8860b482470, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f2691d57c74543df8ecdf0e741c35f96, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/5d79f3f69df6460d9cc8c70bfecd422d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/43979c3918ca45bcb7c012f628a71d13, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b0a4038b40194c758f038b1ed748697d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/18a086d909c4464ab3e89ba80801e85f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/396f2129ca284fc6b8da7328ae68e770, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f0c0f3892a8d4bcb8c37c36f230fd193, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/faa27142946441b4a523e3b49ffec008, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/881e5c7c1c3547eabb5eda8710452d84, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/acf2003d41324c20a66cd592882156ed, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/20485a94c39b4e958d684b5f9487c88f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/8315548c6f914a928dc2fb6d1a1b56b4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/6778e8bb7aa94ec290b21baabe007e92, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/28360282211343eda5f26227742d99e8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/a43c518b8a244db88a0bba0c0eae8d43, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4862a4bea6b343a1a6f8d1650d7e9ff8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4dedee93b62842cbb287512ccc014e4e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/eb4a7f3d291043d58b0d9033036cd28b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/709c82aeb91e432f8101fb6f58a2c30d] to archive 2024-11-08T09:23:34,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:23:34,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/281ed49c36794c9aada9b00e23933ef6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/281ed49c36794c9aada9b00e23933ef6 2024-11-08T09:23:34,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9fb6b1f4568f464a8b65e74d72699edf to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9fb6b1f4568f464a8b65e74d72699edf 2024-11-08T09:23:34,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9972dabfc62142ba9be220a952cbc210 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/9972dabfc62142ba9be220a952cbc210 2024-11-08T09:23:34,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/38418e31c5a84c4d9109d2c9000a3546 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/38418e31c5a84c4d9109d2c9000a3546 2024-11-08T09:23:34,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b960ef53d4754bcc888087a98181dbb9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b960ef53d4754bcc888087a98181dbb9 2024-11-08T09:23:34,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/23fff7216093440d970395ee9b5101c2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/23fff7216093440d970395ee9b5101c2 2024-11-08T09:23:34,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/ba1f9539534c4bf88419a8860b482470 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/ba1f9539534c4bf88419a8860b482470 2024-11-08T09:23:34,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f2691d57c74543df8ecdf0e741c35f96 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f2691d57c74543df8ecdf0e741c35f96 2024-11-08T09:23:34,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/5d79f3f69df6460d9cc8c70bfecd422d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/5d79f3f69df6460d9cc8c70bfecd422d 2024-11-08T09:23:34,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/43979c3918ca45bcb7c012f628a71d13 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/43979c3918ca45bcb7c012f628a71d13 2024-11-08T09:23:34,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b0a4038b40194c758f038b1ed748697d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/b0a4038b40194c758f038b1ed748697d 2024-11-08T09:23:34,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/18a086d909c4464ab3e89ba80801e85f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/18a086d909c4464ab3e89ba80801e85f 2024-11-08T09:23:34,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/396f2129ca284fc6b8da7328ae68e770 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/396f2129ca284fc6b8da7328ae68e770 2024-11-08T09:23:34,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f0c0f3892a8d4bcb8c37c36f230fd193 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f0c0f3892a8d4bcb8c37c36f230fd193 2024-11-08T09:23:34,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/faa27142946441b4a523e3b49ffec008 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/faa27142946441b4a523e3b49ffec008 2024-11-08T09:23:34,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/881e5c7c1c3547eabb5eda8710452d84 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/881e5c7c1c3547eabb5eda8710452d84 2024-11-08T09:23:34,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/acf2003d41324c20a66cd592882156ed to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/acf2003d41324c20a66cd592882156ed 2024-11-08T09:23:34,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/20485a94c39b4e958d684b5f9487c88f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/20485a94c39b4e958d684b5f9487c88f 2024-11-08T09:23:34,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/8315548c6f914a928dc2fb6d1a1b56b4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/8315548c6f914a928dc2fb6d1a1b56b4 2024-11-08T09:23:34,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/6778e8bb7aa94ec290b21baabe007e92 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/6778e8bb7aa94ec290b21baabe007e92 2024-11-08T09:23:34,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/28360282211343eda5f26227742d99e8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/28360282211343eda5f26227742d99e8 2024-11-08T09:23:34,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/a43c518b8a244db88a0bba0c0eae8d43 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/a43c518b8a244db88a0bba0c0eae8d43 2024-11-08T09:23:34,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4862a4bea6b343a1a6f8d1650d7e9ff8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4862a4bea6b343a1a6f8d1650d7e9ff8 2024-11-08T09:23:34,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4dedee93b62842cbb287512ccc014e4e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/4dedee93b62842cbb287512ccc014e4e 2024-11-08T09:23:34,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/eb4a7f3d291043d58b0d9033036cd28b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/eb4a7f3d291043d58b0d9033036cd28b 2024-11-08T09:23:34,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/709c82aeb91e432f8101fb6f58a2c30d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/709c82aeb91e432f8101fb6f58a2c30d 2024-11-08T09:23:34,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/e35b0bad595d45ca89097a938ba577b8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/672d3029a90c419e864ce51380489936, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bbc185884d6b438c8d542cbf943a8482, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6352186285cb49768130450a4d970e3e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/800ea9b7652a48aeb45c673d2b847a24, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d97f52ae3d204210852171fabea1fa9b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/b096154991554b088ebb39e4a6510f36, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9be573efab274d1194e3cdb252d7af2a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9e0003b61ba44e0399edb88b82b7fb55, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bcf1a8a7a5fe4bd38c206ce9e1f2e78f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4e3fb38e874d4e29b9b807fc389f5460, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/06adaab781e6427a8cfe31e10b20356e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/8eba501ef35349b789f715ac04d57ef1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1da4f5508911462eb7d9cd55eca4d249, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ee747d5c505542318e1f008635eb8b31, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/c98e9dcc28bf4825b444f529f0301ce0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4647eb3445a74ce1be742a85eb3afb41, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/296b8962f1554ab08a7343ef23385e49, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/aa66da528aab45e2946b5d408751a31a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/83879346cc2743c4a89e156cb6473af7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/282f00ec0a9c48cb9149d7f1d1bd2d2d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/f996f24f0b76454f8b7f48ae331a64d9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/a728ee4fb34a485ebfe0c21e3ed0fe03, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1e45dbc981b74448b2abbbf3c0c4699d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d01124a4168b47d9a8ca24213b4a0050, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/de6f3ec7842d4b7086827719fa09cbee] to archive 2024-11-08T09:23:34,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:23:34,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/e35b0bad595d45ca89097a938ba577b8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/e35b0bad595d45ca89097a938ba577b8 2024-11-08T09:23:34,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/672d3029a90c419e864ce51380489936 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/672d3029a90c419e864ce51380489936 2024-11-08T09:23:34,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bbc185884d6b438c8d542cbf943a8482 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bbc185884d6b438c8d542cbf943a8482 2024-11-08T09:23:34,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6352186285cb49768130450a4d970e3e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6352186285cb49768130450a4d970e3e 2024-11-08T09:23:34,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/800ea9b7652a48aeb45c673d2b847a24 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/800ea9b7652a48aeb45c673d2b847a24 2024-11-08T09:23:34,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d97f52ae3d204210852171fabea1fa9b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d97f52ae3d204210852171fabea1fa9b 2024-11-08T09:23:34,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/b096154991554b088ebb39e4a6510f36 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/b096154991554b088ebb39e4a6510f36 2024-11-08T09:23:34,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9be573efab274d1194e3cdb252d7af2a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9be573efab274d1194e3cdb252d7af2a 2024-11-08T09:23:34,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9e0003b61ba44e0399edb88b82b7fb55 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/9e0003b61ba44e0399edb88b82b7fb55 2024-11-08T09:23:34,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bcf1a8a7a5fe4bd38c206ce9e1f2e78f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/bcf1a8a7a5fe4bd38c206ce9e1f2e78f 2024-11-08T09:23:34,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4e3fb38e874d4e29b9b807fc389f5460 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4e3fb38e874d4e29b9b807fc389f5460 2024-11-08T09:23:34,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/06adaab781e6427a8cfe31e10b20356e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/06adaab781e6427a8cfe31e10b20356e 2024-11-08T09:23:34,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/8eba501ef35349b789f715ac04d57ef1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/8eba501ef35349b789f715ac04d57ef1 2024-11-08T09:23:34,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1da4f5508911462eb7d9cd55eca4d249 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1da4f5508911462eb7d9cd55eca4d249 2024-11-08T09:23:34,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ee747d5c505542318e1f008635eb8b31 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ee747d5c505542318e1f008635eb8b31 2024-11-08T09:23:34,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/c98e9dcc28bf4825b444f529f0301ce0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/c98e9dcc28bf4825b444f529f0301ce0 2024-11-08T09:23:34,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4647eb3445a74ce1be742a85eb3afb41 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/4647eb3445a74ce1be742a85eb3afb41 2024-11-08T09:23:34,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/296b8962f1554ab08a7343ef23385e49 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/296b8962f1554ab08a7343ef23385e49 2024-11-08T09:23:34,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/aa66da528aab45e2946b5d408751a31a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/aa66da528aab45e2946b5d408751a31a 2024-11-08T09:23:34,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/83879346cc2743c4a89e156cb6473af7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/83879346cc2743c4a89e156cb6473af7 2024-11-08T09:23:34,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/282f00ec0a9c48cb9149d7f1d1bd2d2d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/282f00ec0a9c48cb9149d7f1d1bd2d2d 2024-11-08T09:23:34,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/f996f24f0b76454f8b7f48ae331a64d9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/f996f24f0b76454f8b7f48ae331a64d9 2024-11-08T09:23:34,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/a728ee4fb34a485ebfe0c21e3ed0fe03 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/a728ee4fb34a485ebfe0c21e3ed0fe03 2024-11-08T09:23:34,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1e45dbc981b74448b2abbbf3c0c4699d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/1e45dbc981b74448b2abbbf3c0c4699d 2024-11-08T09:23:34,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d01124a4168b47d9a8ca24213b4a0050 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/d01124a4168b47d9a8ca24213b4a0050 2024-11-08T09:23:34,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/de6f3ec7842d4b7086827719fa09cbee to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/de6f3ec7842d4b7086827719fa09cbee 2024-11-08T09:23:34,277 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/recovered.edits/398.seqid, newMaxSeqId=398, maxSeqId=4 2024-11-08T09:23:34,278 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. 2024-11-08T09:23:34,278 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for e5919dd324cd2c737f9099915b329cd6: Waiting for close lock at 1731057812386Running coprocessor pre-close hooks at 1731057812386Disabling compacts and flushes for region at 1731057812386Disabling writes for close at 1731057812386Obtaining lock to block concurrent updates at 1731057812386Preparing flush snapshotting stores in e5919dd324cd2c737f9099915b329cd6 at 1731057812386Finished memstore snapshotting TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731057812387 (+1 ms)Flushing stores of TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6. at 1731057812388 (+1 ms)Flushing e5919dd324cd2c737f9099915b329cd6/A: creating writer at 1731057812388Flushing mob file e5919dd324cd2c737f9099915b329cd6/A: appending metadata at 1731057812397 (+9 ms)Flushing mob file e5919dd324cd2c737f9099915b329cd6/A: closing flushed file at 1731057812397Flushing e5919dd324cd2c737f9099915b329cd6/A: appending metadata at 1731057812813 (+416 ms)Flushing e5919dd324cd2c737f9099915b329cd6/A: closing flushed file at 1731057812814 (+1 ms)Flushing e5919dd324cd2c737f9099915b329cd6/B: creating writer at 1731057813228 (+414 ms)Flushing e5919dd324cd2c737f9099915b329cd6/B: appending metadata at 1731057813230 (+2 ms)Flushing e5919dd324cd2c737f9099915b329cd6/B: closing flushed file at 1731057813230Flushing e5919dd324cd2c737f9099915b329cd6/C: creating writer at 1731057813762 (+532 ms)Flushing e5919dd324cd2c737f9099915b329cd6/C: appending metadata at 1731057813766 (+4 ms)Flushing e5919dd324cd2c737f9099915b329cd6/C: closing flushed file at 1731057813766Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7986c95c: reopening flushed file at 1731057814182 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e68fbe7: reopening flushed file at 1731057814189 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d5ab583: reopening flushed file at 1731057814193 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e5919dd324cd2c737f9099915b329cd6 in 1811ms, sequenceid=395, compaction requested=true at 1731057814198 (+5 ms)Writing region close event to WAL at 1731057814274 (+76 ms)Running coprocessor post-close hooks at 1731057814278 (+4 ms)Closed at 1731057814278 2024-11-08T09:23:34,280 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,280 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=e5919dd324cd2c737f9099915b329cd6, regionState=CLOSED 2024-11-08T09:23:34,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:23:34,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-08T09:23:34,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure e5919dd324cd2c737f9099915b329cd6, server=cf4d76213909,39151,1731057746690 in 2.0500 sec 2024-11-08T09:23:34,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T09:23:34,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5919dd324cd2c737f9099915b329cd6, UNASSIGN in 2.0540 sec 2024-11-08T09:23:34,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-08T09:23:34,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0570 sec 2024-11-08T09:23:34,287 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057814287"}]},"ts":"1731057814287"} 2024-11-08T09:23:34,288 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T09:23:34,288 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T09:23:34,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0670 sec 2024-11-08T09:23:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T09:23:34,371 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:34,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T09:23:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:34,375 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T09:23:34,376 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:34,380 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,384 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/recovered.edits] 2024-11-08T09:23:34,387 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a4948d074b654fe389277c3e20f8ed46 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/a4948d074b654fe389277c3e20f8ed46 2024-11-08T09:23:34,388 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b0e38a3a2e95454f9b4e868b4de57e2b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/b0e38a3a2e95454f9b4e868b4de57e2b 2024-11-08T09:23:34,389 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0b7011df09944deb01efa029d11e2aa to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/A/c0b7011df09944deb01efa029d11e2aa 2024-11-08T09:23:34,392 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/00f56302f2d2462aa679fedf0b7d4566 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/00f56302f2d2462aa679fedf0b7d4566 2024-11-08T09:23:34,393 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/7e3374ebcbc44f11a67a4b4fcbf658be to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/7e3374ebcbc44f11a67a4b4fcbf658be 2024-11-08T09:23:34,394 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f8449ea13b6e48d48bbe6a84b01b7e6c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/B/f8449ea13b6e48d48bbe6a84b01b7e6c 2024-11-08T09:23:34,397 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6db4b98eed524d598ca4893ca934d099 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/6db4b98eed524d598ca4893ca934d099 2024-11-08T09:23:34,399 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ac31634c539449c087fb12014b236c26 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/ac31634c539449c087fb12014b236c26 2024-11-08T09:23:34,400 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/dd5643b564c94a7a90c232e916e7c452 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/C/dd5643b564c94a7a90c232e916e7c452 2024-11-08T09:23:34,403 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/recovered.edits/398.seqid to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6/recovered.edits/398.seqid 2024-11-08T09:23:34,403 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,404 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T09:23:34,404 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T09:23:34,405 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T09:23:34,409 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110803f4cf96694949fd9f512d0b7392910b_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110803f4cf96694949fd9f512d0b7392910b_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,410 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080a88680ee4ba4b4dac2006b4c455ff2d_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080a88680ee4ba4b4dac2006b4c455ff2d_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,412 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108130ed4d93828477f8560fe4376e90261_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108130ed4d93828477f8560fe4376e90261_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,413 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081728be38b2cf405ea6b74a64dbec8bc3_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081728be38b2cf405ea6b74a64dbec8bc3_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,414 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108191192fda2f545a78a631325ca7a386b_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108191192fda2f545a78a631325ca7a386b_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,415 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108202848e884b84f42ac754ff0405eb368_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108202848e884b84f42ac754ff0405eb368_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,416 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b76bdc6a63c4f50b6abc07baf14bd80_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b76bdc6a63c4f50b6abc07baf14bd80_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,417 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085d5b0639d9cb4d2d814947e565e1c20f_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085d5b0639d9cb4d2d814947e565e1c20f_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,418 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108614661f4e57f4538a2caf2d6eb09d094_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108614661f4e57f4538a2caf2d6eb09d094_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,419 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110881aa03917628408fa95d107f9fc6ba21_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110881aa03917628408fa95d107f9fc6ba21_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,420 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882bb75cec2a741689b8e49aa41055ea2_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882bb75cec2a741689b8e49aa41055ea2_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,421 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110892b598e4feb84c3c83bb26c13e7dd478_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110892b598e4feb84c3c83bb26c13e7dd478_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,422 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108982a8bf11d9249deb6fe0a9125e22865_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108982a8bf11d9249deb6fe0a9125e22865_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,423 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a6844f8003db42f58129d0e034fae2f8_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a6844f8003db42f58129d0e034fae2f8_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,424 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108aed6d3ed0ee24ef79d2e684c2a82ad71_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108aed6d3ed0ee24ef79d2e684c2a82ad71_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,425 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b95ac4138e7a451f9532afa4e9084b28_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b95ac4138e7a451f9532afa4e9084b28_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,426 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b99f89a4de0b4405bbc91703279f24a3_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b99f89a4de0b4405bbc91703279f24a3_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,427 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c8fcfca8890045ba847b2a3b868a9ca0_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c8fcfca8890045ba847b2a3b868a9ca0_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,428 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ef37487361eb4c309a318c7296269c1f_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ef37487361eb4c309a318c7296269c1f_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,429 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f4d52dd790fd4e11b03bdff0f1f58aa4_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f4d52dd790fd4e11b03bdff0f1f58aa4_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,430 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ff20d41021dc4d9d88ed8988bcddbde0_e5919dd324cd2c737f9099915b329cd6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ff20d41021dc4d9d88ed8988bcddbde0_e5919dd324cd2c737f9099915b329cd6 2024-11-08T09:23:34,431 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T09:23:34,433 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:34,435 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T09:23:34,437 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T09:23:34,438 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:34,438 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T09:23:34,439 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731057814438"}]},"ts":"9223372036854775807"} 2024-11-08T09:23:34,443 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T09:23:34,443 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e5919dd324cd2c737f9099915b329cd6, NAME => 'TestAcidGuarantees,,1731057778946.e5919dd324cd2c737f9099915b329cd6.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T09:23:34,443 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T09:23:34,443 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731057814443"}]},"ts":"9223372036854775807"} 2024-11-08T09:23:34,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T09:23:34,446 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:34,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-11-08T09:23:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T09:23:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T09:23:34,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:34,493 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=400 (was 528), ProcessCount=11 (was 11), AvailableMemoryMB=5549 (was 5667) 2024-11-08T09:23:34,501 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=400, ProcessCount=11, AvailableMemoryMB=5548 2024-11-08T09:23:34,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:23:34,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:23:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:34,504 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T09:23:34,504 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:34,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-08T09:23:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T09:23:34,505 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T09:23:34,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742043_1219 (size=959) 2024-11-08T09:23:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T09:23:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T09:23:34,916 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:23:34,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742044_1220 (size=53) 2024-11-08T09:23:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T09:23:35,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:23:35,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 98a1ca1fc4cffd0f4b6116a5b6d6ff56, disabling compactions & flushes 2024-11-08T09:23:35,330 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:35,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:35,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. after waiting 0 ms 2024-11-08T09:23:35,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:35,331 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:35,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: Waiting for close lock at 1731057815330Disabling compacts and flushes for region at 1731057815330Disabling writes for close at 1731057815330Writing region close event to WAL at 1731057815331 (+1 ms)Closed at 1731057815331 2024-11-08T09:23:35,333 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T09:23:35,334 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731057815333"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731057815333"}]},"ts":"1731057815333"} 2024-11-08T09:23:35,336 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T09:23:35,338 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T09:23:35,338 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057815338"}]},"ts":"1731057815338"} 2024-11-08T09:23:35,341 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T09:23:35,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, ASSIGN}] 2024-11-08T09:23:35,343 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, ASSIGN 2024-11-08T09:23:35,344 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, ASSIGN; state=OFFLINE, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=false 2024-11-08T09:23:35,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=98a1ca1fc4cffd0f4b6116a5b6d6ff56, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:35,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, ASSIGN because future has completed 2024-11-08T09:23:35,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:23:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T09:23:35,661 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:35,661 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:23:35,661 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,661 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:23:35,662 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,662 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,664 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,666 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98a1ca1fc4cffd0f4b6116a5b6d6ff56 columnFamilyName A 2024-11-08T09:23:35,666 DEBUG [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:35,667 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] regionserver.HStore(327): Store=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:35,667 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,669 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98a1ca1fc4cffd0f4b6116a5b6d6ff56 columnFamilyName B 2024-11-08T09:23:35,669 DEBUG [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:35,670 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] regionserver.HStore(327): Store=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:35,670 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,672 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98a1ca1fc4cffd0f4b6116a5b6d6ff56 columnFamilyName C 2024-11-08T09:23:35,672 DEBUG [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:23:35,672 INFO [StoreOpener-98a1ca1fc4cffd0f4b6116a5b6d6ff56-1 {}] regionserver.HStore(327): Store=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:23:35,672 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,673 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,674 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,675 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,675 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,676 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:23:35,677 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,680 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:23:35,680 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 98a1ca1fc4cffd0f4b6116a5b6d6ff56; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61092388, jitterRate=-0.08965247869491577}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:23:35,681 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:35,681 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: Running coprocessor pre-open hook at 1731057815662Writing region info on filesystem at 1731057815662Initializing all the Stores at 1731057815663 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057815663Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057815663Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057815664 (+1 ms)Cleaning up temporary data from old regions at 1731057815675 (+11 ms)Running coprocessor post-open hooks at 1731057815681 (+6 ms)Region opened successfully at 1731057815681 2024-11-08T09:23:35,682 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., pid=61, masterSystemTime=1731057815653 2024-11-08T09:23:35,685 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:35,685 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:35,686 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=98a1ca1fc4cffd0f4b6116a5b6d6ff56, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:35,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:23:35,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T09:23:35,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 in 188 msec 2024-11-08T09:23:35,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-08T09:23:35,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, ASSIGN in 348 msec 2024-11-08T09:23:35,692 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T09:23:35,692 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057815692"}]},"ts":"1731057815692"} 2024-11-08T09:23:35,693 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T09:23:35,694 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T09:23:35,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-08T09:23:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T09:23:36,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:36,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c45ccf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e92b4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,664 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab7238d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,665 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,668 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T09:23:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,35807,1731057745600 2024-11-08T09:23:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1674d0fa 2024-11-08T09:23:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T09:23:36,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T09:23:36,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e140226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,672 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbbb94c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1abd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bf308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,676 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,678 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d4b3c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ba0760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,681 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47daf76f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,682 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f106e33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f8f105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51de5242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c8bbd6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaa706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,688 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5c0231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@819cb60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b74fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,692 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a585c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc06976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0a17bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a151db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dac53ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb595da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d9af43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da6fa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262694dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ed7424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7707250f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df95348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:23:36,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:23:36,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:23:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:23:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:23:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1771437f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:23:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:23:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:36,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:23:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea9ca7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:23:36,707 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,707 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,707 DEBUG [Thread-1039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,707 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,707 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:36,708 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,708 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,708 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,708 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-08T09:23:36,709 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:23:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T09:23:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,711 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:36,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:23:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:23:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,713 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:36,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:36,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:23:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:23:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:36,734 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:36,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T09:23:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057876749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057876750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057876752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057876755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057876755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/923611e6b4a34d8598cba5e3366371bc is 50, key is test_row_0/A:col10/1731057816733/Put/seqid=0 2024-11-08T09:23:36,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742045_1221 (size=12001) 2024-11-08T09:23:36,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/923611e6b4a34d8598cba5e3366371bc 2024-11-08T09:23:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T09:23:36,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/94d4e8934af24468acc8d4da4b626646 is 50, key is test_row_0/B:col10/1731057816733/Put/seqid=0 2024-11-08T09:23:36,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057876861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057876862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057876862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:36,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057876872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057876872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:36,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:36,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:36,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742046_1222 (size=12001) 2024-11-08T09:23:36,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/94d4e8934af24468acc8d4da4b626646 2024-11-08T09:23:36,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/c580ca4fb1384442a25d57baf483b7e2 is 50, key is test_row_0/C:col10/1731057816733/Put/seqid=0 2024-11-08T09:23:36,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742047_1223 (size=12001) 2024-11-08T09:23:37,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T09:23:37,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057877072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057877072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057877072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057877081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057877081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:37,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:37,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/c580ca4fb1384442a25d57baf483b7e2 2024-11-08T09:23:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/923611e6b4a34d8598cba5e3366371bc as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/923611e6b4a34d8598cba5e3366371bc 2024-11-08T09:23:37,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/923611e6b4a34d8598cba5e3366371bc, entries=150, sequenceid=20, filesize=11.7 K 2024-11-08T09:23:37,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/94d4e8934af24468acc8d4da4b626646 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/94d4e8934af24468acc8d4da4b626646 2024-11-08T09:23:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T09:23:37,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/94d4e8934af24468acc8d4da4b626646, entries=150, sequenceid=20, filesize=11.7 K 2024-11-08T09:23:37,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/c580ca4fb1384442a25d57baf483b7e2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/c580ca4fb1384442a25d57baf483b7e2 2024-11-08T09:23:37,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/c580ca4fb1384442a25d57baf483b7e2, entries=150, sequenceid=20, filesize=11.7 K 2024-11-08T09:23:37,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 616ms, sequenceid=20, compaction requested=false 2024-11-08T09:23:37,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:37,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:37,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T09:23:37,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/76c955460b2e4bad82ebd17bf3a50b63 is 50, key is test_row_0/A:col10/1731057817384/Put/seqid=0 2024-11-08T09:23:37,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742048_1224 (size=16681) 2024-11-08T09:23:37,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057877399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057877400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057877401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057877401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057877402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:37,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057877511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057877511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057877511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057877512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057877512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:37,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:37,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057877722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057877722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057877722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057877722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057877722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:37,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/76c955460b2e4bad82ebd17bf3a50b63 2024-11-08T09:23:37,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/d258803a2902491ea00445380f1e060b is 50, key is test_row_0/B:col10/1731057817384/Put/seqid=0 2024-11-08T09:23:37,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742049_1225 (size=12001) 2024-11-08T09:23:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T09:23:37,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:37,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057878031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057878031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057878032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057878032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057878032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:38,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:38,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/d258803a2902491ea00445380f1e060b 2024-11-08T09:23:38,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/13d42cdcb874400da177d84e45c3dbe0 is 50, key is test_row_0/C:col10/1731057817384/Put/seqid=0 2024-11-08T09:23:38,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742050_1226 (size=12001) 2024-11-08T09:23:38,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:38,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:38,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:38,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:38,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057878542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057878542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057878542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057878542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057878542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:38,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:38,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:38,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:38,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:38,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/13d42cdcb874400da177d84e45c3dbe0 2024-11-08T09:23:38,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/76c955460b2e4bad82ebd17bf3a50b63 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/76c955460b2e4bad82ebd17bf3a50b63 2024-11-08T09:23:38,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/76c955460b2e4bad82ebd17bf3a50b63, entries=250, sequenceid=40, filesize=16.3 K 2024-11-08T09:23:38,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/d258803a2902491ea00445380f1e060b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d258803a2902491ea00445380f1e060b 2024-11-08T09:23:38,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d258803a2902491ea00445380f1e060b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T09:23:38,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/13d42cdcb874400da177d84e45c3dbe0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/13d42cdcb874400da177d84e45c3dbe0 2024-11-08T09:23:38,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/13d42cdcb874400da177d84e45c3dbe0, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T09:23:38,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1253ms, sequenceid=40, compaction requested=false 2024-11-08T09:23:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:38,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T09:23:38,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:38,704 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T09:23:38,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/85f8d8a41601491ba49370be111497c2 is 50, key is test_row_0/A:col10/1731057817400/Put/seqid=0 2024-11-08T09:23:38,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742051_1227 (size=12001) 2024-11-08T09:23:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T09:23:39,159 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/85f8d8a41601491ba49370be111497c2 2024-11-08T09:23:39,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/6a375f648b904f0983b025588aa528f8 is 50, key is test_row_0/B:col10/1731057817400/Put/seqid=0 2024-11-08T09:23:39,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742052_1228 (size=12001) 2024-11-08T09:23:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:39,571 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/6a375f648b904f0983b025588aa528f8 2024-11-08T09:23:39,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/5888ac2b51f74eaca53c3856b2d9bf90 is 50, key is test_row_0/C:col10/1731057817400/Put/seqid=0 2024-11-08T09:23:39,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742053_1229 (size=12001) 2024-11-08T09:23:39,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057879577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057879576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057879614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057879614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057879614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057879722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057879722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057879722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057879722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057879722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,803 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T09:23:39,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057879932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057879932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057879932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057879932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057879932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:39,991 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/5888ac2b51f74eaca53c3856b2d9bf90 2024-11-08T09:23:39,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/85f8d8a41601491ba49370be111497c2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/85f8d8a41601491ba49370be111497c2 2024-11-08T09:23:40,002 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/85f8d8a41601491ba49370be111497c2, entries=150, sequenceid=57, filesize=11.7 K 2024-11-08T09:23:40,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/6a375f648b904f0983b025588aa528f8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6a375f648b904f0983b025588aa528f8 2024-11-08T09:23:40,008 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6a375f648b904f0983b025588aa528f8, entries=150, sequenceid=57, filesize=11.7 K 2024-11-08T09:23:40,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/5888ac2b51f74eaca53c3856b2d9bf90 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/5888ac2b51f74eaca53c3856b2d9bf90 2024-11-08T09:23:40,013 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/5888ac2b51f74eaca53c3856b2d9bf90, entries=150, sequenceid=57, filesize=11.7 K 2024-11-08T09:23:40,014 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1310ms, sequenceid=57, compaction requested=true 2024-11-08T09:23:40,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:40,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-08T09:23:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-08T09:23:40,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T09:23:40,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3030 sec 2024-11-08T09:23:40,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.3100 sec 2024-11-08T09:23:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:40,243 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:40,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T09:23:40,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/193f2f795c514e35969d6c65d895ebac is 50, key is test_row_0/A:col10/1731057820242/Put/seqid=0 2024-11-08T09:23:40,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742054_1230 (size=12001) 2024-11-08T09:23:40,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057880253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057880257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057880257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057880257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057880258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057880361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057880371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057880371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057880371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057880371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057880571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057880581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057880581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057880581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057880581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/193f2f795c514e35969d6c65d895ebac 2024-11-08T09:23:40,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/77ab34b698484438a84521239f74670c is 50, key is test_row_0/B:col10/1731057820242/Put/seqid=0 2024-11-08T09:23:40,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742055_1231 (size=12001) 2024-11-08T09:23:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T09:23:40,882 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:40,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-08T09:23:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T09:23:40,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:40,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:40,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:40,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057880882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T09:23:41,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T09:23:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/77ab34b698484438a84521239f74670c 2024-11-08T09:23:41,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/fd1b74b0bc9047afaf636f04ccd4708f is 50, key is test_row_0/C:col10/1731057820242/Put/seqid=0 2024-11-08T09:23:41,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742056_1232 (size=12001) 2024-11-08T09:23:41,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T09:23:41,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:41,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T09:23:41,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T09:23:41,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:41,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057881391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057881401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057881401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057881402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:41,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057881402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:41,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/fd1b74b0bc9047afaf636f04ccd4708f 2024-11-08T09:23:41,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/193f2f795c514e35969d6c65d895ebac as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/193f2f795c514e35969d6c65d895ebac 2024-11-08T09:23:41,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T09:23:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:41,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/193f2f795c514e35969d6c65d895ebac, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T09:23:41,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/77ab34b698484438a84521239f74670c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/77ab34b698484438a84521239f74670c 2024-11-08T09:23:41,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/77ab34b698484438a84521239f74670c, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T09:23:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/fd1b74b0bc9047afaf636f04ccd4708f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/fd1b74b0bc9047afaf636f04ccd4708f 2024-11-08T09:23:41,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/fd1b74b0bc9047afaf636f04ccd4708f, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T09:23:41,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1266ms, sequenceid=78, compaction requested=true 2024-11-08T09:23:41,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:41,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:41,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:41,509 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:41,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:41,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:41,509 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:41,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:41,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:41,510 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:41,510 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:41,510 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:41,510 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:41,510 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,510 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,510 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/923611e6b4a34d8598cba5e3366371bc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/76c955460b2e4bad82ebd17bf3a50b63, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/85f8d8a41601491ba49370be111497c2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/193f2f795c514e35969d6c65d895ebac] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=51.4 K 2024-11-08T09:23:41,510 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/94d4e8934af24468acc8d4da4b626646, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d258803a2902491ea00445380f1e060b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6a375f648b904f0983b025588aa528f8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/77ab34b698484438a84521239f74670c] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=46.9 K 2024-11-08T09:23:41,511 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d4e8934af24468acc8d4da4b626646, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1731057816731 2024-11-08T09:23:41,511 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923611e6b4a34d8598cba5e3366371bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1731057816731 2024-11-08T09:23:41,511 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c955460b2e4bad82ebd17bf3a50b63, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731057816752 2024-11-08T09:23:41,511 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d258803a2902491ea00445380f1e060b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731057817382 2024-11-08T09:23:41,511 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85f8d8a41601491ba49370be111497c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731057817398 2024-11-08T09:23:41,511 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a375f648b904f0983b025588aa528f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731057817398 2024-11-08T09:23:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T09:23:41,512 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 193f2f795c514e35969d6c65d895ebac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057819577 2024-11-08T09:23:41,512 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ab34b698484438a84521239f74670c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057819577 2024-11-08T09:23:41,520 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:41,521 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/4c4a33d594114218a96a969a2b805da3 is 50, key is test_row_0/B:col10/1731057820242/Put/seqid=0 2024-11-08T09:23:41,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742057_1233 (size=12139) 2024-11-08T09:23:41,525 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:41,525 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/384fe9d1709142c89a5c4edfc35c8b08 is 50, key is test_row_0/A:col10/1731057820242/Put/seqid=0 2024-11-08T09:23:41,532 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/4c4a33d594114218a96a969a2b805da3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/4c4a33d594114218a96a969a2b805da3 2024-11-08T09:23:41,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742058_1234 (size=12139) 2024-11-08T09:23:41,537 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 4c4a33d594114218a96a969a2b805da3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:41,537 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:41,537 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=12, startTime=1731057821509; duration=0sec 2024-11-08T09:23:41,537 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:41,537 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:41,537 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:23:41,540 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:23:41,540 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C is initiating minor compaction (all files) 2024-11-08T09:23:41,540 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,540 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/c580ca4fb1384442a25d57baf483b7e2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/13d42cdcb874400da177d84e45c3dbe0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/5888ac2b51f74eaca53c3856b2d9bf90, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/fd1b74b0bc9047afaf636f04ccd4708f] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=46.9 K 2024-11-08T09:23:41,541 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting c580ca4fb1384442a25d57baf483b7e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1731057816731 2024-11-08T09:23:41,541 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 13d42cdcb874400da177d84e45c3dbe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731057817382 2024-11-08T09:23:41,541 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5888ac2b51f74eaca53c3856b2d9bf90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731057817398 2024-11-08T09:23:41,541 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1b74b0bc9047afaf636f04ccd4708f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057819577 2024-11-08T09:23:41,550 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:41,550 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/ec113ba2ac4542f0ae321e262d43f364 is 50, key is test_row_0/C:col10/1731057820242/Put/seqid=0 2024-11-08T09:23:41,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742059_1235 (size=12139) 2024-11-08T09:23:41,560 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/ec113ba2ac4542f0ae321e262d43f364 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ec113ba2ac4542f0ae321e262d43f364 2024-11-08T09:23:41,565 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into ec113ba2ac4542f0ae321e262d43f364(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:41,565 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:41,566 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, priority=12, startTime=1731057821509; duration=0sec 2024-11-08T09:23:41,566 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:41,566 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:41,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T09:23:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:41,649 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T09:23:41,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/5604593bbe12471594ae6695f6ba6254 is 50, key is test_row_0/A:col10/1731057820255/Put/seqid=0 2024-11-08T09:23:41,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742060_1236 (size=12001) 2024-11-08T09:23:41,938 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/384fe9d1709142c89a5c4edfc35c8b08 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/384fe9d1709142c89a5c4edfc35c8b08 2024-11-08T09:23:41,943 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 384fe9d1709142c89a5c4edfc35c8b08(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:41,943 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:41,943 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=12, startTime=1731057821509; duration=0sec 2024-11-08T09:23:41,943 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:41,943 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T09:23:42,058 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/5604593bbe12471594ae6695f6ba6254 2024-11-08T09:23:42,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/92ff5f935e974573bd5565861fd92f90 is 50, key is test_row_0/B:col10/1731057820255/Put/seqid=0 2024-11-08T09:23:42,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742061_1237 (size=12001) 2024-11-08T09:23:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:42,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:42,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057882422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,426 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 98a1ca1fc4cffd0f4b6116a5b6d6ff56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057882423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057882424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057882424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057882425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,471 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/92ff5f935e974573bd5565861fd92f90 2024-11-08T09:23:42,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/3319c3270dbb413087f630a2dc4dd75b is 50, key is test_row_0/C:col10/1731057820255/Put/seqid=0 2024-11-08T09:23:42,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742062_1238 (size=12001) 2024-11-08T09:23:42,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057882531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057882532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057882532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057882532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057882741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057882741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057882741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057882742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:42,884 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/3319c3270dbb413087f630a2dc4dd75b 2024-11-08T09:23:42,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/5604593bbe12471594ae6695f6ba6254 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5604593bbe12471594ae6695f6ba6254 2024-11-08T09:23:42,893 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5604593bbe12471594ae6695f6ba6254, entries=150, sequenceid=96, filesize=11.7 K 2024-11-08T09:23:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/92ff5f935e974573bd5565861fd92f90 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92ff5f935e974573bd5565861fd92f90 2024-11-08T09:23:42,898 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92ff5f935e974573bd5565861fd92f90, entries=150, sequenceid=96, filesize=11.7 K 2024-11-08T09:23:42,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/3319c3270dbb413087f630a2dc4dd75b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3319c3270dbb413087f630a2dc4dd75b 2024-11-08T09:23:42,903 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3319c3270dbb413087f630a2dc4dd75b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-08T09:23:42,904 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1255ms, sequenceid=96, compaction requested=false 2024-11-08T09:23:42,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:42,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:42,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-08T09:23:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-08T09:23:42,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T09:23:42,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0210 sec 2024-11-08T09:23:42,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.0260 sec 2024-11-08T09:23:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T09:23:43,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:43,032 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-08T09:23:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T09:23:43,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:43,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:43,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:43,054 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:43,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T09:23:43,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/f8f2152d6c754b25b649c953aa605d2c is 50, key is test_row_0/A:col10/1731057823054/Put/seqid=0 2024-11-08T09:23:43,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742063_1239 (size=12001) 2024-11-08T09:23:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057883067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057883067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057883068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057883069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T09:23:43,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057883182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057883182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057883182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057883182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T09:23:43,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T09:23:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T09:23:43,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057883391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057883392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057883392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057883392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/f8f2152d6c754b25b649c953aa605d2c 2024-11-08T09:23:43,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/76c04d1a7b4b4f9296c02c230f295770 is 50, key is test_row_0/B:col10/1731057823054/Put/seqid=0 2024-11-08T09:23:43,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742064_1240 (size=12001) 2024-11-08T09:23:43,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T09:23:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T09:23:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T09:23:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057883701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057883701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057883701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057883701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:43,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T09:23:43,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:43,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:43,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/76c04d1a7b4b4f9296c02c230f295770 2024-11-08T09:23:43,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/f0f3bd4062aa455c97a7637cba9239f8 is 50, key is test_row_0/C:col10/1731057823054/Put/seqid=0 2024-11-08T09:23:43,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742065_1241 (size=12001) 2024-11-08T09:23:43,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/f0f3bd4062aa455c97a7637cba9239f8 2024-11-08T09:23:43,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/f8f2152d6c754b25b649c953aa605d2c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f2152d6c754b25b649c953aa605d2c 2024-11-08T09:23:43,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f2152d6c754b25b649c953aa605d2c, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T09:23:43,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/76c04d1a7b4b4f9296c02c230f295770 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/76c04d1a7b4b4f9296c02c230f295770 2024-11-08T09:23:43,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/76c04d1a7b4b4f9296c02c230f295770, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T09:23:43,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/f0f3bd4062aa455c97a7637cba9239f8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f0f3bd4062aa455c97a7637cba9239f8 2024-11-08T09:23:43,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f0f3bd4062aa455c97a7637cba9239f8, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T09:23:43,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 861ms, sequenceid=120, compaction requested=true 2024-11-08T09:23:43,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:43,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:43,915 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:43,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:43,915 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:43,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:43,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:43,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:43,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:43,916 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:43,916 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:43,916 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:43,917 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:43,917 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,917 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,917 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/4c4a33d594114218a96a969a2b805da3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92ff5f935e974573bd5565861fd92f90, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/76c04d1a7b4b4f9296c02c230f295770] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.3 K 2024-11-08T09:23:43,917 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/384fe9d1709142c89a5c4edfc35c8b08, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5604593bbe12471594ae6695f6ba6254, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f2152d6c754b25b649c953aa605d2c] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.3 K 2024-11-08T09:23:43,917 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384fe9d1709142c89a5c4edfc35c8b08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057819577 2024-11-08T09:23:43,917 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4a33d594114218a96a969a2b805da3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057819577 2024-11-08T09:23:43,918 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5604593bbe12471594ae6695f6ba6254, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731057820255 2024-11-08T09:23:43,918 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ff5f935e974573bd5565861fd92f90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731057820255 2024-11-08T09:23:43,918 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f2152d6c754b25b649c953aa605d2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731057823052 2024-11-08T09:23:43,918 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c04d1a7b4b4f9296c02c230f295770, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731057823052 2024-11-08T09:23:43,926 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:43,926 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/bf5e66674a254582b01da2660e87044e is 50, key is test_row_0/A:col10/1731057823054/Put/seqid=0 2024-11-08T09:23:43,932 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:43,933 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/6d98123e3b414bbe8dc44e1eed2ba794 is 50, key is test_row_0/B:col10/1731057823054/Put/seqid=0 2024-11-08T09:23:43,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742066_1242 (size=12241) 2024-11-08T09:23:43,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T09:23:43,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,952 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:23:43,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/f8f23a0d91ef4231bd3c7865d02506c3 is 50, key is test_row_0/A:col10/1731057823060/Put/seqid=0 2024-11-08T09:23:43,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742067_1243 (size=12241) 2024-11-08T09:23:43,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742068_1244 (size=12101) 2024-11-08T09:23:43,963 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/f8f23a0d91ef4231bd3c7865d02506c3 2024-11-08T09:23:43,968 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/6d98123e3b414bbe8dc44e1eed2ba794 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6d98123e3b414bbe8dc44e1eed2ba794 2024-11-08T09:23:43,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/7563a05211c8415bba810bf512749a41 is 50, key is test_row_0/B:col10/1731057823060/Put/seqid=0 2024-11-08T09:23:43,977 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 6d98123e3b414bbe8dc44e1eed2ba794(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:43,977 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:43,977 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=13, startTime=1731057823915; duration=0sec 2024-11-08T09:23:43,977 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:43,977 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:43,977 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:43,979 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:43,979 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C is initiating minor compaction (all files) 2024-11-08T09:23:43,979 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:43,979 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ec113ba2ac4542f0ae321e262d43f364, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3319c3270dbb413087f630a2dc4dd75b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f0f3bd4062aa455c97a7637cba9239f8] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.3 K 2024-11-08T09:23:43,979 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting ec113ba2ac4542f0ae321e262d43f364, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057819577 2024-11-08T09:23:43,979 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 3319c3270dbb413087f630a2dc4dd75b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731057820255 2024-11-08T09:23:43,980 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f0f3bd4062aa455c97a7637cba9239f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731057823052 2024-11-08T09:23:43,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742069_1245 (size=12101) 2024-11-08T09:23:43,988 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#C#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:43,988 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/2fa5c3765dcb4c688472025d6e364d0d is 50, key is test_row_0/C:col10/1731057823054/Put/seqid=0 2024-11-08T09:23:43,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742070_1246 (size=12241) 2024-11-08T09:23:43,998 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/2fa5c3765dcb4c688472025d6e364d0d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/2fa5c3765dcb4c688472025d6e364d0d 2024-11-08T09:23:44,003 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 2fa5c3765dcb4c688472025d6e364d0d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:44,003 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:44,003 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, priority=13, startTime=1731057823915; duration=0sec 2024-11-08T09:23:44,003 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:44,004 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T09:23:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:44,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:44,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057884235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057884235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057884240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057884240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,345 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/bf5e66674a254582b01da2660e87044e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bf5e66674a254582b01da2660e87044e 2024-11-08T09:23:44,350 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into bf5e66674a254582b01da2660e87044e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:44,350 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:44,350 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=13, startTime=1731057823915; duration=0sec 2024-11-08T09:23:44,351 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:44,351 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:44,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057884351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057884351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057884351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057884351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,383 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/7563a05211c8415bba810bf512749a41 2024-11-08T09:23:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/4689082da93d4c1a9f5563bf3de1ea33 is 50, key is test_row_0/C:col10/1731057823060/Put/seqid=0 2024-11-08T09:23:44,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742071_1247 (size=12101) 2024-11-08T09:23:44,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057884441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,444 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 98a1ca1fc4cffd0f4b6116a5b6d6ff56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057884561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057884561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057884561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057884561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,798 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/4689082da93d4c1a9f5563bf3de1ea33 2024-11-08T09:23:44,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/f8f23a0d91ef4231bd3c7865d02506c3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f23a0d91ef4231bd3c7865d02506c3 2024-11-08T09:23:44,807 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f23a0d91ef4231bd3c7865d02506c3, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T09:23:44,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/7563a05211c8415bba810bf512749a41 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/7563a05211c8415bba810bf512749a41 2024-11-08T09:23:44,811 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/7563a05211c8415bba810bf512749a41, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T09:23:44,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/4689082da93d4c1a9f5563bf3de1ea33 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/4689082da93d4c1a9f5563bf3de1ea33 2024-11-08T09:23:44,816 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/4689082da93d4c1a9f5563bf3de1ea33, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T09:23:44,817 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 864ms, sequenceid=133, compaction requested=false 2024-11-08T09:23:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-08T09:23:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-08T09:23:44,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T09:23:44,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-08T09:23:44,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7880 sec 2024-11-08T09:23:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:44,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:44,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:23:44,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/c71e018e9d6342789d31b05ae85cd656 is 50, key is test_row_0/A:col10/1731057824239/Put/seqid=0 2024-11-08T09:23:44,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742072_1248 (size=12151) 2024-11-08T09:23:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057884881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057884882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057884883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057884884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057884991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057884991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057884991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057884991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T09:23:45,181 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:45,181 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-08T09:23:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T09:23:45,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:45,184 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:45,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057885201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057885201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057885201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057885201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/c71e018e9d6342789d31b05ae85cd656 2024-11-08T09:23:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T09:23:45,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/723b27830b9d436a9ca4b5a52bda47a8 is 50, key is test_row_0/B:col10/1731057824239/Put/seqid=0 2024-11-08T09:23:45,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742073_1249 (size=12151) 2024-11-08T09:23:45,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T09:23:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T09:23:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T09:23:45,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057885511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057885511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057885511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057885511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:45,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T09:23:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/723b27830b9d436a9ca4b5a52bda47a8 2024-11-08T09:23:45,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/41eda87fbf4a44caa154eb2f4ccbd520 is 50, key is test_row_0/C:col10/1731057824239/Put/seqid=0 2024-11-08T09:23:45,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742074_1250 (size=12151) 2024-11-08T09:23:45,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T09:23:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T09:23:45,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T09:23:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:45,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:45,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:46,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057886021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057886021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057886021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057886021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:46,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:46,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T09:23:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:46,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:46,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:46,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/41eda87fbf4a44caa154eb2f4ccbd520 2024-11-08T09:23:46,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/c71e018e9d6342789d31b05ae85cd656 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/c71e018e9d6342789d31b05ae85cd656 2024-11-08T09:23:46,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/c71e018e9d6342789d31b05ae85cd656, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T09:23:46,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/723b27830b9d436a9ca4b5a52bda47a8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/723b27830b9d436a9ca4b5a52bda47a8 2024-11-08T09:23:46,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/723b27830b9d436a9ca4b5a52bda47a8, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T09:23:46,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/41eda87fbf4a44caa154eb2f4ccbd520 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/41eda87fbf4a44caa154eb2f4ccbd520 2024-11-08T09:23:46,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/41eda87fbf4a44caa154eb2f4ccbd520, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T09:23:46,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1253ms, sequenceid=160, compaction requested=true 2024-11-08T09:23:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:46,126 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:46,126 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:46,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:46,127 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:46,127 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:46,127 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:46,127 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:46,127 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:46,127 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:46,127 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bf5e66674a254582b01da2660e87044e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f23a0d91ef4231bd3c7865d02506c3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/c71e018e9d6342789d31b05ae85cd656] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.6 K 2024-11-08T09:23:46,127 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6d98123e3b414bbe8dc44e1eed2ba794, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/7563a05211c8415bba810bf512749a41, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/723b27830b9d436a9ca4b5a52bda47a8] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.6 K 2024-11-08T09:23:46,128 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5e66674a254582b01da2660e87044e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731057823052 2024-11-08T09:23:46,128 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d98123e3b414bbe8dc44e1eed2ba794, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731057823052 2024-11-08T09:23:46,128 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f23a0d91ef4231bd3c7865d02506c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057823060 2024-11-08T09:23:46,128 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 7563a05211c8415bba810bf512749a41, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057823060 2024-11-08T09:23:46,128 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c71e018e9d6342789d31b05ae85cd656, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731057824233 2024-11-08T09:23:46,128 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 723b27830b9d436a9ca4b5a52bda47a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731057824233 2024-11-08T09:23:46,137 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:46,137 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:46,137 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/da99ba075426486cb51481cd590e34de is 50, key is test_row_0/B:col10/1731057824239/Put/seqid=0 2024-11-08T09:23:46,137 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/bbba2bcb982d41398aeee929a6379a51 is 50, key is test_row_0/A:col10/1731057824239/Put/seqid=0 2024-11-08T09:23:46,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742076_1252 (size=12493) 2024-11-08T09:23:46,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742075_1251 (size=12493) 2024-11-08T09:23:46,154 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/bbba2bcb982d41398aeee929a6379a51 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bbba2bcb982d41398aeee929a6379a51 2024-11-08T09:23:46,160 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into bbba2bcb982d41398aeee929a6379a51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:46,160 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:46,160 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=13, startTime=1731057826126; duration=0sec 2024-11-08T09:23:46,160 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:46,160 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:46,160 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:46,162 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:46,162 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C is initiating minor compaction (all files) 2024-11-08T09:23:46,162 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:46,162 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/2fa5c3765dcb4c688472025d6e364d0d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/4689082da93d4c1a9f5563bf3de1ea33, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/41eda87fbf4a44caa154eb2f4ccbd520] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.6 K 2024-11-08T09:23:46,162 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa5c3765dcb4c688472025d6e364d0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731057823052 2024-11-08T09:23:46,162 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4689082da93d4c1a9f5563bf3de1ea33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057823060 2024-11-08T09:23:46,163 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41eda87fbf4a44caa154eb2f4ccbd520, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731057824233 2024-11-08T09:23:46,169 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:46,169 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/92513b065ea14cf78496ec6767e203b8 is 50, key is test_row_0/C:col10/1731057824239/Put/seqid=0 2024-11-08T09:23:46,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742077_1253 (size=12493) 2024-11-08T09:23:46,177 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/92513b065ea14cf78496ec6767e203b8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/92513b065ea14cf78496ec6767e203b8 2024-11-08T09:23:46,181 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 92513b065ea14cf78496ec6767e203b8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:46,181 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:46,181 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, priority=13, startTime=1731057826126; duration=0sec 2024-11-08T09:23:46,181 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:46,181 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:46,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T09:23:46,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:46,252 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:23:46,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/49fe409af9d246d5bc99654b8a5d79ae is 50, key is test_row_0/A:col10/1731057824879/Put/seqid=0 2024-11-08T09:23:46,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742078_1254 (size=12151) 2024-11-08T09:23:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T09:23:46,551 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/da99ba075426486cb51481cd590e34de as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/da99ba075426486cb51481cd590e34de 2024-11-08T09:23:46,555 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into da99ba075426486cb51481cd590e34de(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:46,555 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:46,555 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=13, startTime=1731057826126; duration=0sec 2024-11-08T09:23:46,555 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:46,555 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:46,661 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/49fe409af9d246d5bc99654b8a5d79ae 2024-11-08T09:23:46,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/5dd7a24bf73c462fb86e9f41693cbc95 is 50, key is test_row_0/B:col10/1731057824879/Put/seqid=0 2024-11-08T09:23:46,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742079_1255 (size=12151) 2024-11-08T09:23:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:47,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:47,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057887054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057887055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057887058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057887058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,083 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/5dd7a24bf73c462fb86e9f41693cbc95 2024-11-08T09:23:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/b180c0f062d74de0b167688cd9e081fd is 50, key is test_row_0/C:col10/1731057824879/Put/seqid=0 2024-11-08T09:23:47,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742080_1256 (size=12151) 2024-11-08T09:23:47,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057887161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057887161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057887171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057887171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T09:23:47,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057887371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057887371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057887381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057887382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,497 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/b180c0f062d74de0b167688cd9e081fd 2024-11-08T09:23:47,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/49fe409af9d246d5bc99654b8a5d79ae as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/49fe409af9d246d5bc99654b8a5d79ae 2024-11-08T09:23:47,506 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/49fe409af9d246d5bc99654b8a5d79ae, entries=150, sequenceid=175, filesize=11.9 K 2024-11-08T09:23:47,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/5dd7a24bf73c462fb86e9f41693cbc95 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/5dd7a24bf73c462fb86e9f41693cbc95 2024-11-08T09:23:47,511 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/5dd7a24bf73c462fb86e9f41693cbc95, entries=150, sequenceid=175, filesize=11.9 K 2024-11-08T09:23:47,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/b180c0f062d74de0b167688cd9e081fd as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b180c0f062d74de0b167688cd9e081fd 2024-11-08T09:23:47,516 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b180c0f062d74de0b167688cd9e081fd, entries=150, sequenceid=175, filesize=11.9 K 2024-11-08T09:23:47,517 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1264ms, sequenceid=175, compaction requested=false 2024-11-08T09:23:47,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:47,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:47,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-08T09:23:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-08T09:23:47,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-08T09:23:47,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-11-08T09:23:47,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3410 sec 2024-11-08T09:23:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:47,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:47,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:23:47,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9e873b256b0a4982a5a3d04c9c71bb52 is 50, key is test_row_0/A:col10/1731057827682/Put/seqid=0 2024-11-08T09:23:47,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742081_1257 (size=14541) 2024-11-08T09:23:47,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9e873b256b0a4982a5a3d04c9c71bb52 2024-11-08T09:23:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057887695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057887696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/e01af437da624fa49deaa09f5c4e43ce is 50, key is test_row_0/B:col10/1731057827682/Put/seqid=0 2024-11-08T09:23:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057887697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057887697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742082_1258 (size=12151) 2024-11-08T09:23:47,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057887801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057887801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057887811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057887812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057888011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057888011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057888021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057888021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/e01af437da624fa49deaa09f5c4e43ce 2024-11-08T09:23:48,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/aa7c8ccbe71043b2b2cb1d36160d4dfe is 50, key is test_row_0/C:col10/1731057827682/Put/seqid=0 2024-11-08T09:23:48,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742083_1259 (size=12151) 2024-11-08T09:23:48,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/aa7c8ccbe71043b2b2cb1d36160d4dfe 2024-11-08T09:23:48,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9e873b256b0a4982a5a3d04c9c71bb52 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9e873b256b0a4982a5a3d04c9c71bb52 2024-11-08T09:23:48,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9e873b256b0a4982a5a3d04c9c71bb52, entries=200, sequenceid=201, filesize=14.2 K 2024-11-08T09:23:48,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/e01af437da624fa49deaa09f5c4e43ce as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e01af437da624fa49deaa09f5c4e43ce 2024-11-08T09:23:48,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e01af437da624fa49deaa09f5c4e43ce, entries=150, sequenceid=201, filesize=11.9 K 2024-11-08T09:23:48,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/aa7c8ccbe71043b2b2cb1d36160d4dfe as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/aa7c8ccbe71043b2b2cb1d36160d4dfe 2024-11-08T09:23:48,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/aa7c8ccbe71043b2b2cb1d36160d4dfe, entries=150, sequenceid=201, filesize=11.9 K 2024-11-08T09:23:48,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 462ms, sequenceid=201, compaction requested=true 2024-11-08T09:23:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:48,145 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:48,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:48,145 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:48,146 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:48,146 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:48,146 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:48,146 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:48,146 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:48,146 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:48,146 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/da99ba075426486cb51481cd590e34de, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/5dd7a24bf73c462fb86e9f41693cbc95, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e01af437da624fa49deaa09f5c4e43ce] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.9 K 2024-11-08T09:23:48,146 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bbba2bcb982d41398aeee929a6379a51, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/49fe409af9d246d5bc99654b8a5d79ae, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9e873b256b0a4982a5a3d04c9c71bb52] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=38.3 K 2024-11-08T09:23:48,147 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting da99ba075426486cb51481cd590e34de, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731057824233 2024-11-08T09:23:48,147 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbba2bcb982d41398aeee929a6379a51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731057824233 2024-11-08T09:23:48,147 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dd7a24bf73c462fb86e9f41693cbc95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731057824879 2024-11-08T09:23:48,147 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49fe409af9d246d5bc99654b8a5d79ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731057824879 2024-11-08T09:23:48,148 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e873b256b0a4982a5a3d04c9c71bb52, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731057827051 2024-11-08T09:23:48,148 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e01af437da624fa49deaa09f5c4e43ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731057827056 2024-11-08T09:23:48,155 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:48,156 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:48,156 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/11611768994947f89e3c740404af543e is 50, key is test_row_0/B:col10/1731057827682/Put/seqid=0 2024-11-08T09:23:48,157 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/7a61ac55fee542f2926c749bf82594cf is 50, key is test_row_0/A:col10/1731057827682/Put/seqid=0 2024-11-08T09:23:48,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742084_1260 (size=12595) 2024-11-08T09:23:48,168 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/11611768994947f89e3c740404af543e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/11611768994947f89e3c740404af543e 2024-11-08T09:23:48,175 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 11611768994947f89e3c740404af543e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:48,175 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:48,175 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=13, startTime=1731057828145; duration=0sec 2024-11-08T09:23:48,175 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:48,175 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:48,175 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:48,177 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:48,177 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C is initiating minor compaction (all files) 2024-11-08T09:23:48,177 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:48,177 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/92513b065ea14cf78496ec6767e203b8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b180c0f062d74de0b167688cd9e081fd, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/aa7c8ccbe71043b2b2cb1d36160d4dfe] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=35.9 K 2024-11-08T09:23:48,178 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 92513b065ea14cf78496ec6767e203b8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731057824233 2024-11-08T09:23:48,178 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b180c0f062d74de0b167688cd9e081fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731057824879 2024-11-08T09:23:48,178 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting aa7c8ccbe71043b2b2cb1d36160d4dfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731057827056 2024-11-08T09:23:48,195 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742085_1261 (size=12595) 2024-11-08T09:23:48,195 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/e4b04d8721054c28a0c0dbe04a5fb714 is 50, key is test_row_0/C:col10/1731057827682/Put/seqid=0 2024-11-08T09:23:48,204 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/7a61ac55fee542f2926c749bf82594cf as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/7a61ac55fee542f2926c749bf82594cf 2024-11-08T09:23:48,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742086_1262 (size=12595) 2024-11-08T09:23:48,211 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 7a61ac55fee542f2926c749bf82594cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:48,211 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:48,212 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=13, startTime=1731057828145; duration=0sec 2024-11-08T09:23:48,212 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:48,212 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:48,323 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:48,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:23:48,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9da4fe2448c348989472e50a10631f7f is 50, key is test_row_0/A:col10/1731057827695/Put/seqid=0 2024-11-08T09:23:48,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742087_1263 (size=14541) 2024-11-08T09:23:48,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9da4fe2448c348989472e50a10631f7f 2024-11-08T09:23:48,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/07e7f9d177c64a75a116149a71539641 is 50, key is test_row_0/B:col10/1731057827695/Put/seqid=0 2024-11-08T09:23:48,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057888351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057888352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057888353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057888354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742088_1264 (size=12151) 2024-11-08T09:23:48,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/07e7f9d177c64a75a116149a71539641 2024-11-08T09:23:48,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/e82ecfc3a8c348d2b1c655f50834f6ce is 50, key is test_row_0/C:col10/1731057827695/Put/seqid=0 2024-11-08T09:23:48,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742089_1265 (size=12151) 2024-11-08T09:23:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057888461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057888461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057888461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057888461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33732 deadline: 1731057888472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,474 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 98a1ca1fc4cffd0f4b6116a5b6d6ff56 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:23:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,616 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/e4b04d8721054c28a0c0dbe04a5fb714 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e4b04d8721054c28a0c0dbe04a5fb714 2024-11-08T09:23:48,621 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into e4b04d8721054c28a0c0dbe04a5fb714(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:48,621 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:48,622 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, priority=13, startTime=1731057828145; duration=0sec 2024-11-08T09:23:48,622 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:48,622 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057888671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057888671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057888671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057888671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/e82ecfc3a8c348d2b1c655f50834f6ce 2024-11-08T09:23:48,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9da4fe2448c348989472e50a10631f7f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9da4fe2448c348989472e50a10631f7f 2024-11-08T09:23:48,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9da4fe2448c348989472e50a10631f7f, entries=200, sequenceid=216, filesize=14.2 K 2024-11-08T09:23:48,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/07e7f9d177c64a75a116149a71539641 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/07e7f9d177c64a75a116149a71539641 2024-11-08T09:23:48,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/07e7f9d177c64a75a116149a71539641, entries=150, sequenceid=216, filesize=11.9 K 2024-11-08T09:23:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/e82ecfc3a8c348d2b1c655f50834f6ce as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e82ecfc3a8c348d2b1c655f50834f6ce 2024-11-08T09:23:48,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e82ecfc3a8c348d2b1c655f50834f6ce, entries=150, sequenceid=216, filesize=11.9 K 2024-11-08T09:23:48,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 476ms, sequenceid=216, compaction requested=false 2024-11-08T09:23:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:48,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:48,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:23:48,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/20a62e4231404162a1247650139d1a43 is 50, key is test_row_0/A:col10/1731057828981/Put/seqid=0 2024-11-08T09:23:48,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742090_1266 (size=12151) 2024-11-08T09:23:48,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057888990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057888992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057888995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057888995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057889101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057889101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057889102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057889102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057889311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057889311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057889311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057889311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T09:23:49,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:49,351 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-08T09:23:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T09:23:49,353 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:49,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:49,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:49,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/20a62e4231404162a1247650139d1a43 2024-11-08T09:23:49,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/44f15954b05d4f3597e4190fce9cd1f5 is 50, key is test_row_0/B:col10/1731057828981/Put/seqid=0 2024-11-08T09:23:49,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742091_1267 (size=12151) 2024-11-08T09:23:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T09:23:49,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T09:23:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057889621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057889621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057889621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057889621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:49,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T09:23:49,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:49,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T09:23:49,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/44f15954b05d4f3597e4190fce9cd1f5 2024-11-08T09:23:49,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T09:23:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/b9bf3bd01ffd47219d92bb4e52eb1cf9 is 50, key is test_row_0/C:col10/1731057828981/Put/seqid=0 2024-11-08T09:23:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742092_1268 (size=12151) 2024-11-08T09:23:49,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T09:23:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:49,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T09:23:50,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T09:23:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:50,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057890131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057890131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057890131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057890131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:50,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/b9bf3bd01ffd47219d92bb4e52eb1cf9 2024-11-08T09:23:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/20a62e4231404162a1247650139d1a43 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/20a62e4231404162a1247650139d1a43 2024-11-08T09:23:50,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/20a62e4231404162a1247650139d1a43, entries=150, sequenceid=241, filesize=11.9 K 2024-11-08T09:23:50,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/44f15954b05d4f3597e4190fce9cd1f5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/44f15954b05d4f3597e4190fce9cd1f5 2024-11-08T09:23:50,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/44f15954b05d4f3597e4190fce9cd1f5, entries=150, sequenceid=241, filesize=11.9 K 2024-11-08T09:23:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/b9bf3bd01ffd47219d92bb4e52eb1cf9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b9bf3bd01ffd47219d92bb4e52eb1cf9 2024-11-08T09:23:50,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b9bf3bd01ffd47219d92bb4e52eb1cf9, entries=150, sequenceid=241, filesize=11.9 K 2024-11-08T09:23:50,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1256ms, sequenceid=241, compaction requested=true 2024-11-08T09:23:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:50,238 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:50,238 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:50,239 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:50,239 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:50,239 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:50,239 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/11611768994947f89e3c740404af543e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/07e7f9d177c64a75a116149a71539641, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/44f15954b05d4f3597e4190fce9cd1f5] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=36.0 K 2024-11-08T09:23:50,240 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 11611768994947f89e3c740404af543e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731057827056 2024-11-08T09:23:50,240 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:50,240 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:50,240 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:50,240 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/7a61ac55fee542f2926c749bf82594cf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9da4fe2448c348989472e50a10631f7f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/20a62e4231404162a1247650139d1a43] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=38.4 K 2024-11-08T09:23:50,240 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e7f9d177c64a75a116149a71539641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731057827694 2024-11-08T09:23:50,240 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a61ac55fee542f2926c749bf82594cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731057827056 2024-11-08T09:23:50,241 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f15954b05d4f3597e4190fce9cd1f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731057828346 2024-11-08T09:23:50,241 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9da4fe2448c348989472e50a10631f7f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731057827694 2024-11-08T09:23:50,241 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a62e4231404162a1247650139d1a43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731057828346 2024-11-08T09:23:50,248 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:50,248 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/eb62a421d8d248f498c460e8dd21961e is 50, key is test_row_0/A:col10/1731057828981/Put/seqid=0 2024-11-08T09:23:50,250 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:50,250 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/0f3a849724cc41d6add6c77a6f7dc571 is 50, key is test_row_0/B:col10/1731057828981/Put/seqid=0 2024-11-08T09:23:50,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742094_1270 (size=12697) 2024-11-08T09:23:50,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742093_1269 (size=12697) 2024-11-08T09:23:50,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T09:23:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:50,270 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:23:50,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/badb295bc4d54bbc910c754f500ea3a5 is 50, key is test_row_0/A:col10/1731057828993/Put/seqid=0 2024-11-08T09:23:50,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742095_1271 (size=9757) 2024-11-08T09:23:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T09:23:50,661 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/0f3a849724cc41d6add6c77a6f7dc571 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/0f3a849724cc41d6add6c77a6f7dc571 2024-11-08T09:23:50,666 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 0f3a849724cc41d6add6c77a6f7dc571(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:50,666 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:50,666 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=13, startTime=1731057830238; duration=0sec 2024-11-08T09:23:50,666 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:50,666 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:50,666 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:50,667 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:50,668 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C is initiating minor compaction (all files) 2024-11-08T09:23:50,668 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:50,668 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e4b04d8721054c28a0c0dbe04a5fb714, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e82ecfc3a8c348d2b1c655f50834f6ce, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b9bf3bd01ffd47219d92bb4e52eb1cf9] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=36.0 K 2024-11-08T09:23:50,668 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b04d8721054c28a0c0dbe04a5fb714, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731057827056 2024-11-08T09:23:50,669 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e82ecfc3a8c348d2b1c655f50834f6ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731057827694 2024-11-08T09:23:50,670 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b9bf3bd01ffd47219d92bb4e52eb1cf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731057828346 2024-11-08T09:23:50,673 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/eb62a421d8d248f498c460e8dd21961e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/eb62a421d8d248f498c460e8dd21961e 2024-11-08T09:23:50,679 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into eb62a421d8d248f498c460e8dd21961e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:50,679 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:50,679 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=13, startTime=1731057830238; duration=0sec 2024-11-08T09:23:50,679 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:50,679 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:50,679 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:50,680 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/bd281c00b30944d188aecc12d8cfebdb is 50, key is test_row_0/C:col10/1731057828981/Put/seqid=0 2024-11-08T09:23:50,680 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/badb295bc4d54bbc910c754f500ea3a5 2024-11-08T09:23:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/ab551846ac144fefbb20ef637ac817eb is 50, key is test_row_0/B:col10/1731057828993/Put/seqid=0 2024-11-08T09:23:50,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742096_1272 (size=12697) 2024-11-08T09:23:50,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742097_1273 (size=9757) 2024-11-08T09:23:51,091 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/bd281c00b30944d188aecc12d8cfebdb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bd281c00b30944d188aecc12d8cfebdb 2024-11-08T09:23:51,097 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into bd281c00b30944d188aecc12d8cfebdb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:51,097 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:51,097 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, priority=13, startTime=1731057830238; duration=0sec 2024-11-08T09:23:51,098 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:51,098 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:51,103 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/ab551846ac144fefbb20ef637ac817eb 2024-11-08T09:23:51,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/cb3b559fb6144234ae188c1dc48b37e1 is 50, key is test_row_0/C:col10/1731057828993/Put/seqid=0 2024-11-08T09:23:51,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742098_1274 (size=9757) 2024-11-08T09:23:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:51,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057891163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057891163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057891164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057891165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057891271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057891271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057891271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057891271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057891481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057891481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057891481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057891481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T09:23:51,527 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/cb3b559fb6144234ae188c1dc48b37e1 2024-11-08T09:23:51,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/badb295bc4d54bbc910c754f500ea3a5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/badb295bc4d54bbc910c754f500ea3a5 2024-11-08T09:23:51,537 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/badb295bc4d54bbc910c754f500ea3a5, entries=100, sequenceid=254, filesize=9.5 K 2024-11-08T09:23:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/ab551846ac144fefbb20ef637ac817eb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ab551846ac144fefbb20ef637ac817eb 2024-11-08T09:23:51,543 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ab551846ac144fefbb20ef637ac817eb, entries=100, sequenceid=254, filesize=9.5 K 2024-11-08T09:23:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/cb3b559fb6144234ae188c1dc48b37e1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/cb3b559fb6144234ae188c1dc48b37e1 2024-11-08T09:23:51,549 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/cb3b559fb6144234ae188c1dc48b37e1, entries=100, sequenceid=254, filesize=9.5 K 2024-11-08T09:23:51,550 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1280ms, sequenceid=254, compaction requested=false 2024-11-08T09:23:51,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:51,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:51,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-08T09:23:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-08T09:23:51,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T09:23:51,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-08T09:23:51,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2070 sec 2024-11-08T09:23:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:51,793 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:51,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:23:51,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/98125e5589f342d3bae352d14f3bb2c6 is 50, key is test_row_0/A:col10/1731057831794/Put/seqid=0 2024-11-08T09:23:51,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742099_1275 (size=12301) 2024-11-08T09:23:51,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057891802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057891803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057891803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057891805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057891911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057891911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057891911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057891911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:51,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057892121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057892121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057892121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057892121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/98125e5589f342d3bae352d14f3bb2c6 2024-11-08T09:23:52,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/ca72317298fd40ca84131c2f05071201 is 50, key is test_row_0/B:col10/1731057831794/Put/seqid=0 2024-11-08T09:23:52,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742100_1276 (size=12301) 2024-11-08T09:23:52,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057892431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057892432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057892432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057892432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/ca72317298fd40ca84131c2f05071201 2024-11-08T09:23:52,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/bcf56ff983214c2e8c01503ea6a1bc79 is 50, key is test_row_0/C:col10/1731057831794/Put/seqid=0 2024-11-08T09:23:52,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742101_1277 (size=12301) 2024-11-08T09:23:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057892941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057892941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057892941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057892942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:52,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:53,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/bcf56ff983214c2e8c01503ea6a1bc79 2024-11-08T09:23:53,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/98125e5589f342d3bae352d14f3bb2c6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/98125e5589f342d3bae352d14f3bb2c6 2024-11-08T09:23:53,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/98125e5589f342d3bae352d14f3bb2c6, entries=150, sequenceid=284, filesize=12.0 K 2024-11-08T09:23:53,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/ca72317298fd40ca84131c2f05071201 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ca72317298fd40ca84131c2f05071201 2024-11-08T09:23:53,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ca72317298fd40ca84131c2f05071201, entries=150, sequenceid=284, filesize=12.0 K 2024-11-08T09:23:53,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/bcf56ff983214c2e8c01503ea6a1bc79 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bcf56ff983214c2e8c01503ea6a1bc79 2024-11-08T09:23:53,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bcf56ff983214c2e8c01503ea6a1bc79, entries=150, sequenceid=284, filesize=12.0 K 2024-11-08T09:23:53,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1260ms, sequenceid=284, compaction requested=true 2024-11-08T09:23:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:53,054 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T09:23:53,054 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:53,055 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:53,055 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:53,055 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:53,055 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/0f3a849724cc41d6add6c77a6f7dc571, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ab551846ac144fefbb20ef637ac817eb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ca72317298fd40ca84131c2f05071201] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=33.9 K 2024-11-08T09:23:53,056 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:53,056 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:53,056 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f3a849724cc41d6add6c77a6f7dc571, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731057828346 2024-11-08T09:23:53,056 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:53,056 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/eb62a421d8d248f498c460e8dd21961e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/badb295bc4d54bbc910c754f500ea3a5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/98125e5589f342d3bae352d14f3bb2c6] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=33.9 K 2024-11-08T09:23:53,056 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab551846ac144fefbb20ef637ac817eb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731057828993 2024-11-08T09:23:53,057 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca72317298fd40ca84131c2f05071201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057831161 2024-11-08T09:23:53,057 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting eb62a421d8d248f498c460e8dd21961e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731057828346 2024-11-08T09:23:53,057 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting badb295bc4d54bbc910c754f500ea3a5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731057828993 2024-11-08T09:23:53,057 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 98125e5589f342d3bae352d14f3bb2c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057831161 2024-11-08T09:23:53,065 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:53,065 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/e58d966a893b43a7841d29f7ab4c7abf is 50, key is test_row_0/B:col10/1731057831794/Put/seqid=0 2024-11-08T09:23:53,069 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:53,069 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/555c392a112f4cf8ad5052721eec3c1b is 50, key is test_row_0/A:col10/1731057831794/Put/seqid=0 2024-11-08T09:23:53,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742102_1278 (size=12949) 2024-11-08T09:23:53,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742103_1279 (size=12949) 2024-11-08T09:23:53,090 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/555c392a112f4cf8ad5052721eec3c1b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/555c392a112f4cf8ad5052721eec3c1b 2024-11-08T09:23:53,096 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 555c392a112f4cf8ad5052721eec3c1b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:53,096 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:53,096 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=13, startTime=1731057833054; duration=0sec 2024-11-08T09:23:53,096 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:53,096 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:53,096 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:53,098 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:53,098 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C is initiating minor compaction (all files) 2024-11-08T09:23:53,098 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:53,098 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bd281c00b30944d188aecc12d8cfebdb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/cb3b559fb6144234ae188c1dc48b37e1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bcf56ff983214c2e8c01503ea6a1bc79] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=33.9 K 2024-11-08T09:23:53,098 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting bd281c00b30944d188aecc12d8cfebdb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731057828346 2024-11-08T09:23:53,099 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3b559fb6144234ae188c1dc48b37e1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731057828993 2024-11-08T09:23:53,099 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf56ff983214c2e8c01503ea6a1bc79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057831161 2024-11-08T09:23:53,108 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:53,108 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/94389534c7b44973b29ade84a9b060d4 is 50, key is test_row_0/C:col10/1731057831794/Put/seqid=0 2024-11-08T09:23:53,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742104_1280 (size=12949) 2024-11-08T09:23:53,131 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/94389534c7b44973b29ade84a9b060d4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/94389534c7b44973b29ade84a9b060d4 2024-11-08T09:23:53,136 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 94389534c7b44973b29ade84a9b060d4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:53,136 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:53,136 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, priority=13, startTime=1731057833054; duration=0sec 2024-11-08T09:23:53,136 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:53,136 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:53,479 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/e58d966a893b43a7841d29f7ab4c7abf as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e58d966a893b43a7841d29f7ab4c7abf 2024-11-08T09:23:53,483 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into e58d966a893b43a7841d29f7ab4c7abf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:53,483 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:53,484 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=13, startTime=1731057833054; duration=0sec 2024-11-08T09:23:53,484 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:53,484 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T09:23:53,511 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:53,512 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-08T09:23:53,513 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:53,514 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:53,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T09:23:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T09:23:53,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T09:23:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:53,667 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T09:23:53,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/28cce2a3166b40a8a1cf42c701219f7b is 50, key is test_row_0/A:col10/1731057831797/Put/seqid=0 2024-11-08T09:23:53,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742105_1281 (size=12301) 2024-11-08T09:23:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T09:23:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:53,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057893976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:53,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:53,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057893978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:53,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057893979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:53,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057893980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,076 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/28cce2a3166b40a8a1cf42c701219f7b 2024-11-08T09:23:54,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057894081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057894091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/bf6deab871254bd2b05197659f1ee9d7 is 50, key is test_row_0/B:col10/1731057831797/Put/seqid=0 2024-11-08T09:23:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057894091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057894091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742106_1282 (size=12301) 2024-11-08T09:23:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T09:23:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057894291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057894301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057894301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057894301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,497 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/bf6deab871254bd2b05197659f1ee9d7 2024-11-08T09:23:54,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/f50580625cb241709bbe3e2864d11a0d is 50, key is test_row_0/C:col10/1731057831797/Put/seqid=0 2024-11-08T09:23:54,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742107_1283 (size=12301) 2024-11-08T09:23:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057894601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057894612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057894612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057894612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T09:23:54,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T09:23:54,909 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/f50580625cb241709bbe3e2864d11a0d 2024-11-08T09:23:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/28cce2a3166b40a8a1cf42c701219f7b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/28cce2a3166b40a8a1cf42c701219f7b 2024-11-08T09:23:54,917 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/28cce2a3166b40a8a1cf42c701219f7b, entries=150, sequenceid=297, filesize=12.0 K 2024-11-08T09:23:54,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/bf6deab871254bd2b05197659f1ee9d7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/bf6deab871254bd2b05197659f1ee9d7 2024-11-08T09:23:54,921 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/bf6deab871254bd2b05197659f1ee9d7, entries=150, sequenceid=297, filesize=12.0 K 2024-11-08T09:23:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/f50580625cb241709bbe3e2864d11a0d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f50580625cb241709bbe3e2864d11a0d 2024-11-08T09:23:54,926 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f50580625cb241709bbe3e2864d11a0d, entries=150, sequenceid=297, filesize=12.0 K 2024-11-08T09:23:54,927 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1260ms, sequenceid=297, compaction requested=false 2024-11-08T09:23:54,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:54,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:54,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-08T09:23:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-08T09:23:54,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T09:23:54,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-08T09:23:54,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4190 sec 2024-11-08T09:23:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:55,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:55,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T09:23:55,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/b24b189e67924f0191cd78e07188a6a1 is 50, key is test_row_0/A:col10/1731057833976/Put/seqid=0 2024-11-08T09:23:55,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742108_1284 (size=14741) 2024-11-08T09:23:55,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057895127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057895131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057895131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057895132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057895241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057895241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057895241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057895242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057895451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057895451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057895451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057895451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/b24b189e67924f0191cd78e07188a6a1 2024-11-08T09:23:55,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/a02f851f48b14df8be2592fb6e0286b1 is 50, key is test_row_0/B:col10/1731057833976/Put/seqid=0 2024-11-08T09:23:55,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742109_1285 (size=12301) 2024-11-08T09:23:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T09:23:55,661 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:55,662 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:23:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-08T09:23:55,663 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:23:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T09:23:55,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:23:55,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:23:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057895761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057895761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057895761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057895761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T09:23:55,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T09:23:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:55,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:55,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:55,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/a02f851f48b14df8be2592fb6e0286b1 2024-11-08T09:23:55,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/783c9f1818fc4268a78ecc9762438e22 is 50, key is test_row_0/C:col10/1731057833976/Put/seqid=0 2024-11-08T09:23:55,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742110_1286 (size=12301) 2024-11-08T09:23:55,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/783c9f1818fc4268a78ecc9762438e22 2024-11-08T09:23:55,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/b24b189e67924f0191cd78e07188a6a1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/b24b189e67924f0191cd78e07188a6a1 2024-11-08T09:23:55,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/b24b189e67924f0191cd78e07188a6a1, entries=200, sequenceid=324, filesize=14.4 K 2024-11-08T09:23:55,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/a02f851f48b14df8be2592fb6e0286b1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a02f851f48b14df8be2592fb6e0286b1 2024-11-08T09:23:55,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a02f851f48b14df8be2592fb6e0286b1, entries=150, sequenceid=324, filesize=12.0 K 2024-11-08T09:23:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/783c9f1818fc4268a78ecc9762438e22 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/783c9f1818fc4268a78ecc9762438e22 2024-11-08T09:23:55,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T09:23:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:55,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:55,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:23:55,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/783c9f1818fc4268a78ecc9762438e22, entries=150, sequenceid=324, filesize=12.0 K 2024-11-08T09:23:55,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 862ms, sequenceid=324, compaction requested=true 2024-11-08T09:23:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:55,977 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:55,977 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:55,978 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:55,978 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:55,978 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:55,978 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/555c392a112f4cf8ad5052721eec3c1b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/28cce2a3166b40a8a1cf42c701219f7b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/b24b189e67924f0191cd78e07188a6a1] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=39.1 K 2024-11-08T09:23:55,978 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:55,978 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:55,978 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:55,978 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 555c392a112f4cf8ad5052721eec3c1b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057831161 2024-11-08T09:23:55,979 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e58d966a893b43a7841d29f7ab4c7abf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/bf6deab871254bd2b05197659f1ee9d7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a02f851f48b14df8be2592fb6e0286b1] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=36.7 K 2024-11-08T09:23:55,979 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 28cce2a3166b40a8a1cf42c701219f7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731057831797 2024-11-08T09:23:55,979 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e58d966a893b43a7841d29f7ab4c7abf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057831161 2024-11-08T09:23:55,979 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b24b189e67924f0191cd78e07188a6a1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057833974 2024-11-08T09:23:55,979 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6deab871254bd2b05197659f1ee9d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731057831797 2024-11-08T09:23:55,979 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting a02f851f48b14df8be2592fb6e0286b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057833974 2024-11-08T09:23:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T09:23:55,986 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:55,986 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:55,987 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/a36fea1ff2d34b54ae906a6a30667831 is 50, key is test_row_0/B:col10/1731057833976/Put/seqid=0 2024-11-08T09:23:55,987 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/ab7a850be9e14b7ab522938362d70b70 is 50, key is test_row_0/A:col10/1731057833976/Put/seqid=0 2024-11-08T09:23:55,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742111_1287 (size=13051) 2024-11-08T09:23:55,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742112_1288 (size=13051) 2024-11-08T09:23:55,997 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/a36fea1ff2d34b54ae906a6a30667831 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a36fea1ff2d34b54ae906a6a30667831 2024-11-08T09:23:56,000 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/ab7a850be9e14b7ab522938362d70b70 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/ab7a850be9e14b7ab522938362d70b70 2024-11-08T09:23:56,002 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into a36fea1ff2d34b54ae906a6a30667831(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:56,002 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:56,002 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=13, startTime=1731057835977; duration=0sec 2024-11-08T09:23:56,002 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:56,003 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:56,003 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:56,004 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:56,004 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C is initiating minor compaction (all files) 2024-11-08T09:23:56,004 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:56,004 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/94389534c7b44973b29ade84a9b060d4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f50580625cb241709bbe3e2864d11a0d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/783c9f1818fc4268a78ecc9762438e22] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=36.7 K 2024-11-08T09:23:56,005 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94389534c7b44973b29ade84a9b060d4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057831161 2024-11-08T09:23:56,005 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f50580625cb241709bbe3e2864d11a0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731057831797 2024-11-08T09:23:56,005 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 783c9f1818fc4268a78ecc9762438e22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057833974 2024-11-08T09:23:56,006 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into ab7a850be9e14b7ab522938362d70b70(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:56,006 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:56,006 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=13, startTime=1731057835977; duration=0sec 2024-11-08T09:23:56,006 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:56,006 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:56,012 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:56,013 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/3885a7dd894a4225b6a3fcb9049e0785 is 50, key is test_row_0/C:col10/1731057833976/Put/seqid=0 2024-11-08T09:23:56,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742113_1289 (size=13051) 2024-11-08T09:23:56,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T09:23:56,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:56,128 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T09:23:56,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9222c81eb5fa4eeb893d526255d4eb3f is 50, key is test_row_0/A:col10/1731057835130/Put/seqid=0 2024-11-08T09:23:56,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742114_1290 (size=12301) 2024-11-08T09:23:56,136 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9222c81eb5fa4eeb893d526255d4eb3f 2024-11-08T09:23:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/d62d7017b7924aff9afd938dfb63f96b is 50, key is test_row_0/B:col10/1731057835130/Put/seqid=0 2024-11-08T09:23:56,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742115_1291 (size=12301) 2024-11-08T09:23:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:56,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. as already flushing 2024-11-08T09:23:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T09:23:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057896295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057896296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057896297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057896298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057896412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057896412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057896412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057896412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,423 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/3885a7dd894a4225b6a3fcb9049e0785 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3885a7dd894a4225b6a3fcb9049e0785 2024-11-08T09:23:56,427 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 3885a7dd894a4225b6a3fcb9049e0785(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:56,427 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:56,427 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, priority=13, startTime=1731057835977; duration=0sec 2024-11-08T09:23:56,427 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:56,428 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:56,549 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/d62d7017b7924aff9afd938dfb63f96b 2024-11-08T09:23:56,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/d47694a590d44d2f84dcd3008e07a2e4 is 50, key is test_row_0/C:col10/1731057835130/Put/seqid=0 2024-11-08T09:23:56,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742116_1292 (size=12301) 2024-11-08T09:23:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057896621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057896621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057896622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057896622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,710 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-08T09:23:56,710 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:56,710 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,710 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,710 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-08T09:23:56,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:56,711 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:56,711 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,711 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:56,712 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-08T09:23:56,712 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:56,712 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,712 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:56,713 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-08T09:23:56,713 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:56,713 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,713 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:56,713 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-11-08T09:23:56,713 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:56,713 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,714 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:56,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T09:23:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33698 deadline: 1731057896933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33726 deadline: 1731057896932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33714 deadline: 1731057896933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33782 deadline: 1731057896933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:23:56,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:23:56,960 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/d47694a590d44d2f84dcd3008e07a2e4 2024-11-08T09:23:56,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/9222c81eb5fa4eeb893d526255d4eb3f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9222c81eb5fa4eeb893d526255d4eb3f 2024-11-08T09:23:56,969 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9222c81eb5fa4eeb893d526255d4eb3f, entries=150, sequenceid=336, filesize=12.0 K 2024-11-08T09:23:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/d62d7017b7924aff9afd938dfb63f96b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d62d7017b7924aff9afd938dfb63f96b 2024-11-08T09:23:56,973 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d62d7017b7924aff9afd938dfb63f96b, entries=150, sequenceid=336, filesize=12.0 K 2024-11-08T09:23:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/d47694a590d44d2f84dcd3008e07a2e4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/d47694a590d44d2f84dcd3008e07a2e4 2024-11-08T09:23:56,978 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/d47694a590d44d2f84dcd3008e07a2e4, entries=150, sequenceid=336, filesize=12.0 K 2024-11-08T09:23:56,978 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 851ms, sequenceid=336, compaction requested=false 2024-11-08T09:23:56,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-08T09:23:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-08T09:23:56,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-08T09:23:56,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3160 sec 2024-11-08T09:23:56,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3200 sec 2024-11-08T09:23:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:57,448 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:23:57,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T09:23:57,448 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-11-08T09:23:57,448 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-08T09:23:57,448 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-11-08T09:23:57,448 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:57,448 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:57,448 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,448 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,448 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:57,448 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,448 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,448 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:57,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:57,448 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:57,455 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-08T09:23:57,455 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:57,455 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,455 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:57,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:57,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/5756b509dff7430cab97233751a5a776 is 50, key is test_row_0/A:col10/1731057836292/Put/seqid=0 2024-11-08T09:23:57,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742117_1293 (size=12301) 2024-11-08T09:23:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T09:23:57,813 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:23:57,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/5756b509dff7430cab97233751a5a776 2024-11-08T09:23:57,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/92e669700816484392c4af7d78df5366 is 50, key is test_row_0/B:col10/1731057836292/Put/seqid=0 2024-11-08T09:23:57,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742118_1294 (size=12301) 2024-11-08T09:23:58,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/92e669700816484392c4af7d78df5366 2024-11-08T09:23:58,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/9e52d30c93f444baa67e17f7ea6bcce0 is 50, key is test_row_0/C:col10/1731057836292/Put/seqid=0 2024-11-08T09:23:58,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742119_1295 (size=12301) 2024-11-08T09:23:58,586 INFO [Thread-1039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1039. 2024-11-08T09:23:58,587 DEBUG [Thread-1039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:23:58,587 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:58,587 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:58,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:58,587 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5102 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4973 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4709 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5090 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4959 2024-11-08T09:23:58,588 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T09:23:58,588 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:23:58,589 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:23:58,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:58,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:23:58,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:23:58,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T09:23:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T09:23:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T09:23:58,596 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057838595"}]},"ts":"1731057838595"} 2024-11-08T09:23:58,597 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T09:23:58,597 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T09:23:58,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:23:58,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, UNASSIGN}] 2024-11-08T09:23:58,601 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, UNASSIGN 2024-11-08T09:23:58,602 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=98a1ca1fc4cffd0f4b6116a5b6d6ff56, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:23:58,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, UNASSIGN because future has completed 2024-11-08T09:23:58,604 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:23:58,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:23:58,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/9e52d30c93f444baa67e17f7ea6bcce0 2024-11-08T09:23:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/5756b509dff7430cab97233751a5a776 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5756b509dff7430cab97233751a5a776 2024-11-08T09:23:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T09:23:58,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5756b509dff7430cab97233751a5a776, entries=150, sequenceid=366, filesize=12.0 K 2024-11-08T09:23:58,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/92e669700816484392c4af7d78df5366 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92e669700816484392c4af7d78df5366 2024-11-08T09:23:58,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92e669700816484392c4af7d78df5366, entries=150, sequenceid=366, filesize=12.0 K 2024-11-08T09:23:58,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/9e52d30c93f444baa67e17f7ea6bcce0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/9e52d30c93f444baa67e17f7ea6bcce0 2024-11-08T09:23:58,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/9e52d30c93f444baa67e17f7ea6bcce0, entries=150, sequenceid=366, filesize=12.0 K 2024-11-08T09:23:58,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=13.42 KB/13740 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1263ms, sequenceid=366, compaction requested=true 2024-11-08T09:23:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:23:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:23:58,711 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:23:58,711 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:23:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:58,712 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:58,712 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:23:58,712 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A is initiating minor compaction (all files) 2024-11-08T09:23:58,712 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B is initiating minor compaction (all files) 2024-11-08T09:23:58,712 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:58,712 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A in TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:58,712 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a36fea1ff2d34b54ae906a6a30667831, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d62d7017b7924aff9afd938dfb63f96b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92e669700816484392c4af7d78df5366] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=36.8 K 2024-11-08T09:23:58,712 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/ab7a850be9e14b7ab522938362d70b70, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9222c81eb5fa4eeb893d526255d4eb3f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5756b509dff7430cab97233751a5a776] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp, totalSize=36.8 K 2024-11-08T09:23:58,712 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a36fea1ff2d34b54ae906a6a30667831, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057833974 2024-11-08T09:23:58,712 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab7a850be9e14b7ab522938362d70b70, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057833974 2024-11-08T09:23:58,713 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9222c81eb5fa4eeb893d526255d4eb3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731057835129 2024-11-08T09:23:58,713 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d62d7017b7924aff9afd938dfb63f96b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731057835129 2024-11-08T09:23:58,713 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5756b509dff7430cab97233751a5a776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731057836292 2024-11-08T09:23:58,713 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e669700816484392c4af7d78df5366, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731057836292 2024-11-08T09:23:58,720 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:58,720 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98a1ca1fc4cffd0f4b6116a5b6d6ff56#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:23:58,721 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/35541ff03edc42f8af14d7f4502936ef is 50, key is test_row_0/B:col10/1731057836292/Put/seqid=0 2024-11-08T09:23:58,721 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/1c19a87fcdf14772a1885f272461747c is 50, key is test_row_0/A:col10/1731057836292/Put/seqid=0 2024-11-08T09:23:58,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742120_1296 (size=13153) 2024-11-08T09:23:58,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742121_1297 (size=13153) 2024-11-08T09:23:58,756 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:23:58,756 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:23:58,756 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 98a1ca1fc4cffd0f4b6116a5b6d6ff56, disabling compactions & flushes 2024-11-08T09:23:58,757 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T09:23:59,174 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/1c19a87fcdf14772a1885f272461747c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/1c19a87fcdf14772a1885f272461747c 2024-11-08T09:23:59,174 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/35541ff03edc42f8af14d7f4502936ef as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/35541ff03edc42f8af14d7f4502936ef 2024-11-08T09:23:59,181 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 1c19a87fcdf14772a1885f272461747c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:59,181 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:59,182 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, priority=13, startTime=1731057838711; duration=0sec 2024-11-08T09:23:59,182 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B of 98a1ca1fc4cffd0f4b6116a5b6d6ff56 into 35541ff03edc42f8af14d7f4502936ef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:23:59,182 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: 2024-11-08T09:23:59,182 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:23:59,182 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:A 2024-11-08T09:23:59,182 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:59,182 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., storeName=98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, priority=13, startTime=1731057838711; duration=0sec 2024-11-08T09:23:59,182 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:59,182 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. after waiting 0 ms 2024-11-08T09:23:59,182 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:23:59,182 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:B 2024-11-08T09:23:59,182 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:23:59,182 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. because compaction request was cancelled 2024-11-08T09:23:59,182 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98a1ca1fc4cffd0f4b6116a5b6d6ff56:C 2024-11-08T09:23:59,182 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-08T09:23:59,185 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/0b902e603f4a46858571f54a89245efe is 50, key is test_row_0/A:col10/1731057837449/Put/seqid=0 2024-11-08T09:23:59,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742122_1298 (size=9857) 2024-11-08T09:23:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T09:23:59,590 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/0b902e603f4a46858571f54a89245efe 2024-11-08T09:23:59,606 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/c92ec0b56f0d4fcbbdc19d158fd92b7a is 50, key is test_row_0/B:col10/1731057837449/Put/seqid=0 2024-11-08T09:23:59,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742123_1299 (size=9857) 2024-11-08T09:23:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T09:24:00,011 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/c92ec0b56f0d4fcbbdc19d158fd92b7a 2024-11-08T09:24:00,024 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/ba0a8e19acfb40a2b7b770aa35688694 is 50, key is test_row_0/C:col10/1731057837449/Put/seqid=0 2024-11-08T09:24:00,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742124_1300 (size=9857) 2024-11-08T09:24:00,430 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/ba0a8e19acfb40a2b7b770aa35688694 2024-11-08T09:24:00,442 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/A/0b902e603f4a46858571f54a89245efe as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/0b902e603f4a46858571f54a89245efe 2024-11-08T09:24:00,447 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/0b902e603f4a46858571f54a89245efe, entries=100, sequenceid=373, filesize=9.6 K 2024-11-08T09:24:00,448 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/B/c92ec0b56f0d4fcbbdc19d158fd92b7a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/c92ec0b56f0d4fcbbdc19d158fd92b7a 2024-11-08T09:24:00,452 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/c92ec0b56f0d4fcbbdc19d158fd92b7a, entries=100, sequenceid=373, filesize=9.6 K 2024-11-08T09:24:00,453 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/.tmp/C/ba0a8e19acfb40a2b7b770aa35688694 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ba0a8e19acfb40a2b7b770aa35688694 2024-11-08T09:24:00,457 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ba0a8e19acfb40a2b7b770aa35688694, entries=100, sequenceid=373, filesize=9.6 K 2024-11-08T09:24:00,458 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1276ms, sequenceid=373, compaction requested=true 2024-11-08T09:24:00,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/923611e6b4a34d8598cba5e3366371bc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/76c955460b2e4bad82ebd17bf3a50b63, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/85f8d8a41601491ba49370be111497c2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/384fe9d1709142c89a5c4edfc35c8b08, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/193f2f795c514e35969d6c65d895ebac, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5604593bbe12471594ae6695f6ba6254, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bf5e66674a254582b01da2660e87044e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f2152d6c754b25b649c953aa605d2c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f23a0d91ef4231bd3c7865d02506c3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bbba2bcb982d41398aeee929a6379a51, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/c71e018e9d6342789d31b05ae85cd656, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/49fe409af9d246d5bc99654b8a5d79ae, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9e873b256b0a4982a5a3d04c9c71bb52, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/7a61ac55fee542f2926c749bf82594cf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9da4fe2448c348989472e50a10631f7f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/eb62a421d8d248f498c460e8dd21961e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/20a62e4231404162a1247650139d1a43, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/badb295bc4d54bbc910c754f500ea3a5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/555c392a112f4cf8ad5052721eec3c1b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/98125e5589f342d3bae352d14f3bb2c6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/28cce2a3166b40a8a1cf42c701219f7b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/b24b189e67924f0191cd78e07188a6a1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/ab7a850be9e14b7ab522938362d70b70, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9222c81eb5fa4eeb893d526255d4eb3f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5756b509dff7430cab97233751a5a776] to archive 2024-11-08T09:24:00,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:24:00,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/923611e6b4a34d8598cba5e3366371bc to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/923611e6b4a34d8598cba5e3366371bc 2024-11-08T09:24:00,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/76c955460b2e4bad82ebd17bf3a50b63 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/76c955460b2e4bad82ebd17bf3a50b63 2024-11-08T09:24:00,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/85f8d8a41601491ba49370be111497c2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/85f8d8a41601491ba49370be111497c2 2024-11-08T09:24:00,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/384fe9d1709142c89a5c4edfc35c8b08 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/384fe9d1709142c89a5c4edfc35c8b08 2024-11-08T09:24:00,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/193f2f795c514e35969d6c65d895ebac to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/193f2f795c514e35969d6c65d895ebac 2024-11-08T09:24:00,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5604593bbe12471594ae6695f6ba6254 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5604593bbe12471594ae6695f6ba6254 2024-11-08T09:24:00,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bf5e66674a254582b01da2660e87044e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bf5e66674a254582b01da2660e87044e 2024-11-08T09:24:00,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f2152d6c754b25b649c953aa605d2c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f2152d6c754b25b649c953aa605d2c 2024-11-08T09:24:00,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f23a0d91ef4231bd3c7865d02506c3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/f8f23a0d91ef4231bd3c7865d02506c3 2024-11-08T09:24:00,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bbba2bcb982d41398aeee929a6379a51 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/bbba2bcb982d41398aeee929a6379a51 2024-11-08T09:24:00,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/c71e018e9d6342789d31b05ae85cd656 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/c71e018e9d6342789d31b05ae85cd656 2024-11-08T09:24:00,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/49fe409af9d246d5bc99654b8a5d79ae to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/49fe409af9d246d5bc99654b8a5d79ae 2024-11-08T09:24:00,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9e873b256b0a4982a5a3d04c9c71bb52 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9e873b256b0a4982a5a3d04c9c71bb52 2024-11-08T09:24:00,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/7a61ac55fee542f2926c749bf82594cf to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/7a61ac55fee542f2926c749bf82594cf 2024-11-08T09:24:00,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9da4fe2448c348989472e50a10631f7f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9da4fe2448c348989472e50a10631f7f 2024-11-08T09:24:00,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/eb62a421d8d248f498c460e8dd21961e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/eb62a421d8d248f498c460e8dd21961e 2024-11-08T09:24:00,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/20a62e4231404162a1247650139d1a43 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/20a62e4231404162a1247650139d1a43 2024-11-08T09:24:00,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/badb295bc4d54bbc910c754f500ea3a5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/badb295bc4d54bbc910c754f500ea3a5 2024-11-08T09:24:00,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/555c392a112f4cf8ad5052721eec3c1b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/555c392a112f4cf8ad5052721eec3c1b 2024-11-08T09:24:00,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/98125e5589f342d3bae352d14f3bb2c6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/98125e5589f342d3bae352d14f3bb2c6 2024-11-08T09:24:00,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/28cce2a3166b40a8a1cf42c701219f7b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/28cce2a3166b40a8a1cf42c701219f7b 2024-11-08T09:24:00,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/b24b189e67924f0191cd78e07188a6a1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/b24b189e67924f0191cd78e07188a6a1 2024-11-08T09:24:00,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/ab7a850be9e14b7ab522938362d70b70 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/ab7a850be9e14b7ab522938362d70b70 2024-11-08T09:24:00,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9222c81eb5fa4eeb893d526255d4eb3f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/9222c81eb5fa4eeb893d526255d4eb3f 2024-11-08T09:24:00,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5756b509dff7430cab97233751a5a776 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/5756b509dff7430cab97233751a5a776 2024-11-08T09:24:00,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/94d4e8934af24468acc8d4da4b626646, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d258803a2902491ea00445380f1e060b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6a375f648b904f0983b025588aa528f8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/4c4a33d594114218a96a969a2b805da3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/77ab34b698484438a84521239f74670c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92ff5f935e974573bd5565861fd92f90, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6d98123e3b414bbe8dc44e1eed2ba794, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/76c04d1a7b4b4f9296c02c230f295770, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/7563a05211c8415bba810bf512749a41, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/da99ba075426486cb51481cd590e34de, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/723b27830b9d436a9ca4b5a52bda47a8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/5dd7a24bf73c462fb86e9f41693cbc95, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/11611768994947f89e3c740404af543e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e01af437da624fa49deaa09f5c4e43ce, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/07e7f9d177c64a75a116149a71539641, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/0f3a849724cc41d6add6c77a6f7dc571, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/44f15954b05d4f3597e4190fce9cd1f5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ab551846ac144fefbb20ef637ac817eb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e58d966a893b43a7841d29f7ab4c7abf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ca72317298fd40ca84131c2f05071201, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/bf6deab871254bd2b05197659f1ee9d7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a36fea1ff2d34b54ae906a6a30667831, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a02f851f48b14df8be2592fb6e0286b1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d62d7017b7924aff9afd938dfb63f96b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92e669700816484392c4af7d78df5366] to archive 2024-11-08T09:24:00,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:24:00,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/94d4e8934af24468acc8d4da4b626646 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/94d4e8934af24468acc8d4da4b626646 2024-11-08T09:24:00,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d258803a2902491ea00445380f1e060b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d258803a2902491ea00445380f1e060b 2024-11-08T09:24:00,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6a375f648b904f0983b025588aa528f8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6a375f648b904f0983b025588aa528f8 2024-11-08T09:24:00,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/4c4a33d594114218a96a969a2b805da3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/4c4a33d594114218a96a969a2b805da3 2024-11-08T09:24:00,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/77ab34b698484438a84521239f74670c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/77ab34b698484438a84521239f74670c 2024-11-08T09:24:00,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92ff5f935e974573bd5565861fd92f90 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92ff5f935e974573bd5565861fd92f90 2024-11-08T09:24:00,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6d98123e3b414bbe8dc44e1eed2ba794 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/6d98123e3b414bbe8dc44e1eed2ba794 2024-11-08T09:24:00,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/76c04d1a7b4b4f9296c02c230f295770 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/76c04d1a7b4b4f9296c02c230f295770 2024-11-08T09:24:00,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/7563a05211c8415bba810bf512749a41 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/7563a05211c8415bba810bf512749a41 2024-11-08T09:24:00,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/da99ba075426486cb51481cd590e34de to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/da99ba075426486cb51481cd590e34de 2024-11-08T09:24:00,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/723b27830b9d436a9ca4b5a52bda47a8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/723b27830b9d436a9ca4b5a52bda47a8 2024-11-08T09:24:00,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/5dd7a24bf73c462fb86e9f41693cbc95 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/5dd7a24bf73c462fb86e9f41693cbc95 2024-11-08T09:24:00,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/11611768994947f89e3c740404af543e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/11611768994947f89e3c740404af543e 2024-11-08T09:24:00,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e01af437da624fa49deaa09f5c4e43ce to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e01af437da624fa49deaa09f5c4e43ce 2024-11-08T09:24:00,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/07e7f9d177c64a75a116149a71539641 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/07e7f9d177c64a75a116149a71539641 2024-11-08T09:24:00,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/0f3a849724cc41d6add6c77a6f7dc571 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/0f3a849724cc41d6add6c77a6f7dc571 2024-11-08T09:24:00,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/44f15954b05d4f3597e4190fce9cd1f5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/44f15954b05d4f3597e4190fce9cd1f5 2024-11-08T09:24:00,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ab551846ac144fefbb20ef637ac817eb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ab551846ac144fefbb20ef637ac817eb 2024-11-08T09:24:00,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e58d966a893b43a7841d29f7ab4c7abf to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/e58d966a893b43a7841d29f7ab4c7abf 2024-11-08T09:24:00,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ca72317298fd40ca84131c2f05071201 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/ca72317298fd40ca84131c2f05071201 2024-11-08T09:24:00,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/bf6deab871254bd2b05197659f1ee9d7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/bf6deab871254bd2b05197659f1ee9d7 2024-11-08T09:24:00,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a36fea1ff2d34b54ae906a6a30667831 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a36fea1ff2d34b54ae906a6a30667831 2024-11-08T09:24:00,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a02f851f48b14df8be2592fb6e0286b1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/a02f851f48b14df8be2592fb6e0286b1 2024-11-08T09:24:00,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d62d7017b7924aff9afd938dfb63f96b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/d62d7017b7924aff9afd938dfb63f96b 2024-11-08T09:24:00,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92e669700816484392c4af7d78df5366 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/92e669700816484392c4af7d78df5366 2024-11-08T09:24:00,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/c580ca4fb1384442a25d57baf483b7e2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/13d42cdcb874400da177d84e45c3dbe0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/5888ac2b51f74eaca53c3856b2d9bf90, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ec113ba2ac4542f0ae321e262d43f364, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/fd1b74b0bc9047afaf636f04ccd4708f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3319c3270dbb413087f630a2dc4dd75b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/2fa5c3765dcb4c688472025d6e364d0d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f0f3bd4062aa455c97a7637cba9239f8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/4689082da93d4c1a9f5563bf3de1ea33, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/92513b065ea14cf78496ec6767e203b8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/41eda87fbf4a44caa154eb2f4ccbd520, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b180c0f062d74de0b167688cd9e081fd, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e4b04d8721054c28a0c0dbe04a5fb714, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/aa7c8ccbe71043b2b2cb1d36160d4dfe, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e82ecfc3a8c348d2b1c655f50834f6ce, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bd281c00b30944d188aecc12d8cfebdb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b9bf3bd01ffd47219d92bb4e52eb1cf9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/cb3b559fb6144234ae188c1dc48b37e1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/94389534c7b44973b29ade84a9b060d4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bcf56ff983214c2e8c01503ea6a1bc79, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f50580625cb241709bbe3e2864d11a0d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/783c9f1818fc4268a78ecc9762438e22] to archive 2024-11-08T09:24:00,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:24:00,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/c580ca4fb1384442a25d57baf483b7e2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/c580ca4fb1384442a25d57baf483b7e2 2024-11-08T09:24:00,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/13d42cdcb874400da177d84e45c3dbe0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/13d42cdcb874400da177d84e45c3dbe0 2024-11-08T09:24:00,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/5888ac2b51f74eaca53c3856b2d9bf90 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/5888ac2b51f74eaca53c3856b2d9bf90 2024-11-08T09:24:00,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ec113ba2ac4542f0ae321e262d43f364 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ec113ba2ac4542f0ae321e262d43f364 2024-11-08T09:24:00,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/fd1b74b0bc9047afaf636f04ccd4708f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/fd1b74b0bc9047afaf636f04ccd4708f 2024-11-08T09:24:00,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3319c3270dbb413087f630a2dc4dd75b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3319c3270dbb413087f630a2dc4dd75b 2024-11-08T09:24:00,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/2fa5c3765dcb4c688472025d6e364d0d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/2fa5c3765dcb4c688472025d6e364d0d 2024-11-08T09:24:00,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f0f3bd4062aa455c97a7637cba9239f8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f0f3bd4062aa455c97a7637cba9239f8 2024-11-08T09:24:00,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/4689082da93d4c1a9f5563bf3de1ea33 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/4689082da93d4c1a9f5563bf3de1ea33 2024-11-08T09:24:00,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/92513b065ea14cf78496ec6767e203b8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/92513b065ea14cf78496ec6767e203b8 2024-11-08T09:24:00,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/41eda87fbf4a44caa154eb2f4ccbd520 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/41eda87fbf4a44caa154eb2f4ccbd520 2024-11-08T09:24:00,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b180c0f062d74de0b167688cd9e081fd to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b180c0f062d74de0b167688cd9e081fd 2024-11-08T09:24:00,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e4b04d8721054c28a0c0dbe04a5fb714 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e4b04d8721054c28a0c0dbe04a5fb714 2024-11-08T09:24:00,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/aa7c8ccbe71043b2b2cb1d36160d4dfe to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/aa7c8ccbe71043b2b2cb1d36160d4dfe 2024-11-08T09:24:00,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e82ecfc3a8c348d2b1c655f50834f6ce to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/e82ecfc3a8c348d2b1c655f50834f6ce 2024-11-08T09:24:00,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bd281c00b30944d188aecc12d8cfebdb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bd281c00b30944d188aecc12d8cfebdb 2024-11-08T09:24:00,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b9bf3bd01ffd47219d92bb4e52eb1cf9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/b9bf3bd01ffd47219d92bb4e52eb1cf9 2024-11-08T09:24:00,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/cb3b559fb6144234ae188c1dc48b37e1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/cb3b559fb6144234ae188c1dc48b37e1 2024-11-08T09:24:00,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/94389534c7b44973b29ade84a9b060d4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/94389534c7b44973b29ade84a9b060d4 2024-11-08T09:24:00,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bcf56ff983214c2e8c01503ea6a1bc79 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/bcf56ff983214c2e8c01503ea6a1bc79 2024-11-08T09:24:00,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f50580625cb241709bbe3e2864d11a0d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/f50580625cb241709bbe3e2864d11a0d 2024-11-08T09:24:00,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/783c9f1818fc4268a78ecc9762438e22 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/783c9f1818fc4268a78ecc9762438e22 2024-11-08T09:24:00,529 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-08T09:24:00,529 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. 2024-11-08T09:24:00,530 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 98a1ca1fc4cffd0f4b6116a5b6d6ff56: Waiting for close lock at 1731057838756Running coprocessor pre-close hooks at 1731057838756Disabling compacts and flushes for region at 1731057838756Disabling writes for close at 1731057839182 (+426 ms)Obtaining lock to block concurrent updates at 1731057839182Preparing flush snapshotting stores in 98a1ca1fc4cffd0f4b6116a5b6d6ff56 at 1731057839182Finished memstore snapshotting TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731057839182Flushing stores of TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56. at 1731057839183 (+1 ms)Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A: creating writer at 1731057839183Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A: appending metadata at 1731057839185 (+2 ms)Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/A: closing flushed file at 1731057839185Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B: creating writer at 1731057839600 (+415 ms)Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B: appending metadata at 1731057839605 (+5 ms)Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/B: closing flushed file at 1731057839605Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C: creating writer at 1731057840020 (+415 ms)Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C: appending metadata at 1731057840023 (+3 ms)Flushing 98a1ca1fc4cffd0f4b6116a5b6d6ff56/C: closing flushed file at 1731057840023Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b417e15: reopening flushed file at 1731057840441 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c346904: reopening flushed file at 1731057840448 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3af82a14: reopening flushed file at 1731057840452 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 98a1ca1fc4cffd0f4b6116a5b6d6ff56 in 1276ms, sequenceid=373, compaction requested=true at 1731057840458 (+6 ms)Writing region close event to WAL at 1731057840527 (+69 ms)Running coprocessor post-close hooks at 1731057840529 (+2 ms)Closed at 1731057840529 2024-11-08T09:24:00,531 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:24:00,532 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=98a1ca1fc4cffd0f4b6116a5b6d6ff56, regionState=CLOSED 2024-11-08T09:24:00,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:24:00,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T09:24:00,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 98a1ca1fc4cffd0f4b6116a5b6d6ff56, server=cf4d76213909,39151,1731057746690 in 1.9300 sec 2024-11-08T09:24:00,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-08T09:24:00,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98a1ca1fc4cffd0f4b6116a5b6d6ff56, UNASSIGN in 1.9360 sec 2024-11-08T09:24:00,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T09:24:00,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-08T09:24:00,542 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057840541"}]},"ts":"1731057840541"} 2024-11-08T09:24:00,543 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T09:24:00,543 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T09:24:00,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9530 sec 2024-11-08T09:24:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T09:24:00,743 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:00,744 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T09:24:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:00,748 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T09:24:00,749 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:00,752 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:24:00,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/recovered.edits] 2024-11-08T09:24:00,758 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/0b902e603f4a46858571f54a89245efe to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/0b902e603f4a46858571f54a89245efe 2024-11-08T09:24:00,759 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/1c19a87fcdf14772a1885f272461747c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/A/1c19a87fcdf14772a1885f272461747c 2024-11-08T09:24:00,761 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/35541ff03edc42f8af14d7f4502936ef to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/35541ff03edc42f8af14d7f4502936ef 2024-11-08T09:24:00,762 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/c92ec0b56f0d4fcbbdc19d158fd92b7a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/B/c92ec0b56f0d4fcbbdc19d158fd92b7a 2024-11-08T09:24:00,764 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3885a7dd894a4225b6a3fcb9049e0785 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/3885a7dd894a4225b6a3fcb9049e0785 2024-11-08T09:24:00,765 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/9e52d30c93f444baa67e17f7ea6bcce0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/9e52d30c93f444baa67e17f7ea6bcce0 2024-11-08T09:24:00,766 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ba0a8e19acfb40a2b7b770aa35688694 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/ba0a8e19acfb40a2b7b770aa35688694 2024-11-08T09:24:00,766 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/d47694a590d44d2f84dcd3008e07a2e4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/C/d47694a590d44d2f84dcd3008e07a2e4 2024-11-08T09:24:00,768 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/recovered.edits/376.seqid to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56/recovered.edits/376.seqid 2024-11-08T09:24:00,769 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/98a1ca1fc4cffd0f4b6116a5b6d6ff56 2024-11-08T09:24:00,769 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T09:24:00,771 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:00,774 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T09:24:00,775 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T09:24:00,776 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:00,776 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T09:24:00,777 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731057840776"}]},"ts":"9223372036854775807"} 2024-11-08T09:24:00,778 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T09:24:00,778 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 98a1ca1fc4cffd0f4b6116a5b6d6ff56, NAME => 'TestAcidGuarantees,,1731057814502.98a1ca1fc4cffd0f4b6116a5b6d6ff56.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T09:24:00,778 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T09:24:00,778 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731057840778"}]},"ts":"9223372036854775807"} 2024-11-08T09:24:00,780 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T09:24:00,780 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:00,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-11-08T09:24:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T09:24:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T09:24:00,852 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:00,862 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_93588470_22 at /127.0.0.1:42332 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2017930777_22 at /127.0.0.1:42334 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2017930777_22 at /127.0.0.1:60058 [Waiting for operation #314] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_93588470_22 at /127.0.0.1:45546 [Waiting for operation #168] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 400), ProcessCount=11 (was 11), AvailableMemoryMB=5499 (was 5548) 2024-11-08T09:24:00,872 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=5499 2024-11-08T09:24:00,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:24:00,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:24:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:00,877 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T09:24:00,877 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:00,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-08T09:24:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-08T09:24:00,878 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T09:24:00,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742125_1301 (size=959) 2024-11-08T09:24:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-08T09:24:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-08T09:24:01,288 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:24:01,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742126_1302 (size=53) 2024-11-08T09:24:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-08T09:24:01,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:24:01,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 22a585182af10d799ec85bb33e624e65, disabling compactions & flushes 2024-11-08T09:24:01,697 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:01,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:01,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. after waiting 0 ms 2024-11-08T09:24:01,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:01,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:01,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 22a585182af10d799ec85bb33e624e65: Waiting for close lock at 1731057841697Disabling compacts and flushes for region at 1731057841697Disabling writes for close at 1731057841698 (+1 ms)Writing region close event to WAL at 1731057841698Closed at 1731057841698 2024-11-08T09:24:01,699 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T09:24:01,700 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731057841699"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731057841699"}]},"ts":"1731057841699"} 2024-11-08T09:24:01,702 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T09:24:01,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T09:24:01,703 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057841703"}]},"ts":"1731057841703"} 2024-11-08T09:24:01,704 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T09:24:01,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, ASSIGN}] 2024-11-08T09:24:01,706 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, ASSIGN 2024-11-08T09:24:01,707 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, ASSIGN; state=OFFLINE, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=false 2024-11-08T09:24:01,858 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:01,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, ASSIGN because future has completed 2024-11-08T09:24:01,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:24:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-08T09:24:02,016 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:02,016 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:24:02,016 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,016 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:24:02,016 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,016 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,018 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,019 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22a585182af10d799ec85bb33e624e65 columnFamilyName A 2024-11-08T09:24:02,019 DEBUG [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:02,020 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(327): Store=22a585182af10d799ec85bb33e624e65/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:02,020 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,021 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22a585182af10d799ec85bb33e624e65 columnFamilyName B 2024-11-08T09:24:02,021 DEBUG [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:02,022 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(327): Store=22a585182af10d799ec85bb33e624e65/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:02,022 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,023 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22a585182af10d799ec85bb33e624e65 columnFamilyName C 2024-11-08T09:24:02,023 DEBUG [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:02,024 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(327): Store=22a585182af10d799ec85bb33e624e65/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:02,024 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,024 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,025 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,026 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,026 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,026 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:24:02,027 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,028 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:24:02,029 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 22a585182af10d799ec85bb33e624e65; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66602620, jitterRate=-0.007543623447418213}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:24:02,029 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:02,030 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 22a585182af10d799ec85bb33e624e65: Running coprocessor pre-open hook at 1731057842016Writing region info on filesystem at 1731057842016Initializing all the Stores at 1731057842017 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057842017Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057842018 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057842018Cleaning up temporary data from old regions at 1731057842026 (+8 ms)Running coprocessor post-open hooks at 1731057842029 (+3 ms)Region opened successfully at 1731057842029 2024-11-08T09:24:02,030 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., pid=83, masterSystemTime=1731057842012 2024-11-08T09:24:02,032 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:02,032 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:02,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:02,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:24:02,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T09:24:02,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 in 174 msec 2024-11-08T09:24:02,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-08T09:24:02,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, ASSIGN in 331 msec 2024-11-08T09:24:02,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T09:24:02,039 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057842038"}]},"ts":"1731057842038"} 2024-11-08T09:24:02,040 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T09:24:02,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T09:24:02,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-08T09:24:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-08T09:24:03,022 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:03,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab4b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c83b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:03,026 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:03,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cee1669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,027 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:03,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:03,029 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T09:24:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,35807,1731057745600 2024-11-08T09:24:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a6ad611 2024-11-08T09:24:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T09:24:03,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T09:24:03,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:24:03,031 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:24:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742127_1303 (size=995) 2024-11-08T09:24:03,445 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T09:24:03,445 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T09:24:03,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:24:03,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, REOPEN/MOVE}] 2024-11-08T09:24:03,455 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, REOPEN/MOVE 2024-11-08T09:24:03,456 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:03,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, REOPEN/MOVE because future has completed 2024-11-08T09:24:03,458 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:24:03,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:24:03,611 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,612 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:24:03,612 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 22a585182af10d799ec85bb33e624e65, disabling compactions & flushes 2024-11-08T09:24:03,612 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:03,613 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:03,613 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. after waiting 0 ms 2024-11-08T09:24:03,613 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:03,623 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T09:24:03,624 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:03,624 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 22a585182af10d799ec85bb33e624e65: Waiting for close lock at 1731057843612Running coprocessor pre-close hooks at 1731057843612Disabling compacts and flushes for region at 1731057843612Disabling writes for close at 1731057843613 (+1 ms)Writing region close event to WAL at 1731057843614 (+1 ms)Running coprocessor post-close hooks at 1731057843623 (+9 ms)Closed at 1731057843623 2024-11-08T09:24:03,624 WARN [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 22a585182af10d799ec85bb33e624e65 to self. 2024-11-08T09:24:03,626 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,626 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=CLOSED 2024-11-08T09:24:03,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:24:03,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-08T09:24:03,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 in 170 msec 2024-11-08T09:24:03,631 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, REOPEN/MOVE; state=CLOSED, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=true 2024-11-08T09:24:03,782 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:03,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, REOPEN/MOVE because future has completed 2024-11-08T09:24:03,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:24:03,950 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:03,950 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:24:03,950 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,950 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:24:03,950 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,950 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,952 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,953 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22a585182af10d799ec85bb33e624e65 columnFamilyName A 2024-11-08T09:24:03,954 DEBUG [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:03,954 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(327): Store=22a585182af10d799ec85bb33e624e65/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:03,955 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,956 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22a585182af10d799ec85bb33e624e65 columnFamilyName B 2024-11-08T09:24:03,956 DEBUG [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:03,956 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(327): Store=22a585182af10d799ec85bb33e624e65/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:03,956 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,957 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22a585182af10d799ec85bb33e624e65 columnFamilyName C 2024-11-08T09:24:03,957 DEBUG [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:03,958 INFO [StoreOpener-22a585182af10d799ec85bb33e624e65-1 {}] regionserver.HStore(327): Store=22a585182af10d799ec85bb33e624e65/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:03,958 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,959 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,960 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,962 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,962 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,962 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:24:03,964 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,966 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 22a585182af10d799ec85bb33e624e65; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62446108, jitterRate=-0.06948047876358032}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:24:03,966 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:03,967 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 22a585182af10d799ec85bb33e624e65: Running coprocessor pre-open hook at 1731057843951Writing region info on filesystem at 1731057843951Initializing all the Stores at 1731057843951Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057843951Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057843952 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057843952Cleaning up temporary data from old regions at 1731057843962 (+10 ms)Running coprocessor post-open hooks at 1731057843966 (+4 ms)Region opened successfully at 1731057843967 (+1 ms) 2024-11-08T09:24:03,968 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., pid=88, masterSystemTime=1731057843941 2024-11-08T09:24:03,970 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:03,970 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:03,971 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=OPEN, openSeqNum=5, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:03,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:24:03,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-08T09:24:03,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 in 186 msec 2024-11-08T09:24:03,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T09:24:03,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, REOPEN/MOVE in 520 msec 2024-11-08T09:24:03,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-08T09:24:03,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-08T09:24:03,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-11-08T09:24:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T09:24:03,980 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:03,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173aa1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,981 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:03,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5464c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e8400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@660d979e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:03,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7e0656, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7eae17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba3b91b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:03,988 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acec2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c5be4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,989 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e03ca13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:03,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c00e8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50946179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,992 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4added19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:03,993 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f9fca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61351bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,995 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459d3112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:03,996 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596a11e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6444e4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:03,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:03,998 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582d624d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:04,000 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24827b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b350ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:04,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:04,001 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffa866f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:04,003 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217b1540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775a762e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:04,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:04,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b89e1e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:04,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:04,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4859f9b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b15f64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:04,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:04,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667a4451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:04,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9dbbd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:04,012 DEBUG [Thread-1405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,012 DEBUG [Thread-1420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,012 DEBUG [Thread-1409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,012 DEBUG [Thread-1407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,013 DEBUG [Thread-1403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,013 DEBUG [Thread-1407 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,013 DEBUG [Thread-1401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,013 DEBUG [Thread-1407 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,013 DEBUG [Thread-1416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,014 DEBUG [Thread-1412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-08T09:24:04,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,016 DEBUG [Thread-1418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,016 DEBUG [Thread-1414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:04,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:04,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T09:24:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:04,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:04,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:04,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:24:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:04,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:24:04,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057904051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057904053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f2b40d0282f542a4b8f1cdd6c402dbec_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057844028/Put/seqid=0 2024-11-08T09:24:04,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057904054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057904056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057904058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742128_1304 (size=12154) 2024-11-08T09:24:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T09:24:04,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:04,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:04,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057904172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057904172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057904172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057904172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057904172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:04,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:04,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T09:24:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057904382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057904382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057904382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057904382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057904383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,470 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:04,475 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f2b40d0282f542a4b8f1cdd6c402dbec_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f2b40d0282f542a4b8f1cdd6c402dbec_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:04,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/adede391982e45e2beb4825e20642ab5, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:04,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/adede391982e45e2beb4825e20642ab5 is 175, key is test_row_0/A:col10/1731057844028/Put/seqid=0 2024-11-08T09:24:04,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:04,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:04,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742129_1305 (size=30955) 2024-11-08T09:24:04,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/adede391982e45e2beb4825e20642ab5 2024-11-08T09:24:04,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/63a986687f4a4856a6e726728a401e0d is 50, key is test_row_0/B:col10/1731057844028/Put/seqid=0 2024-11-08T09:24:04,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742130_1306 (size=12001) 2024-11-08T09:24:04,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/63a986687f4a4856a6e726728a401e0d 2024-11-08T09:24:04,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/261112dc5623407a98949654f4bf899e is 50, key is test_row_0/C:col10/1731057844028/Put/seqid=0 2024-11-08T09:24:04,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742131_1307 (size=12001) 2024-11-08T09:24:04,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/261112dc5623407a98949654f4bf899e 2024-11-08T09:24:04,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/adede391982e45e2beb4825e20642ab5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/adede391982e45e2beb4825e20642ab5 2024-11-08T09:24:04,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/adede391982e45e2beb4825e20642ab5, entries=150, sequenceid=17, filesize=30.2 K 2024-11-08T09:24:04,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/63a986687f4a4856a6e726728a401e0d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/63a986687f4a4856a6e726728a401e0d 2024-11-08T09:24:04,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T09:24:04,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/63a986687f4a4856a6e726728a401e0d, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T09:24:04,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/261112dc5623407a98949654f4bf899e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/261112dc5623407a98949654f4bf899e 2024-11-08T09:24:04,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/261112dc5623407a98949654f4bf899e, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T09:24:04,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 22a585182af10d799ec85bb33e624e65 in 623ms, sequenceid=17, compaction requested=false 2024-11-08T09:24:04,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:04,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:24:04,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086ff0e71b9ee94cc699e6adc3926ddb42_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057844694/Put/seqid=0 2024-11-08T09:24:04,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057904708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057904709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057904710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057904711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057904711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742132_1308 (size=17034) 2024-11-08T09:24:04,725 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:04,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086ff0e71b9ee94cc699e6adc3926ddb42_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086ff0e71b9ee94cc699e6adc3926ddb42_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:04,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/6edc116f08884985932592d0c060d470, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:04,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/6edc116f08884985932592d0c060d470 is 175, key is test_row_0/A:col10/1731057844694/Put/seqid=0 2024-11-08T09:24:04,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742133_1309 (size=48139) 2024-11-08T09:24:04,746 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/6edc116f08884985932592d0c060d470 2024-11-08T09:24:04,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/ada18ebd465a414c9ae4025b6458e1cb is 50, key is test_row_0/B:col10/1731057844694/Put/seqid=0 2024-11-08T09:24:04,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742134_1310 (size=12001) 2024-11-08T09:24:04,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/ada18ebd465a414c9ae4025b6458e1cb 2024-11-08T09:24:04,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c80bb41a27ba46e5b3c077fcc8ef3ff2 is 50, key is test_row_0/C:col10/1731057844694/Put/seqid=0 2024-11-08T09:24:04,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742135_1311 (size=12001) 2024-11-08T09:24:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057904821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057904821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057904822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057904832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057904832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:04,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:04,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:05,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057905041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057905042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057905042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057905051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057905051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:05,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:05,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:05,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:05,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:05,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T09:24:05,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c80bb41a27ba46e5b3c077fcc8ef3ff2 2024-11-08T09:24:05,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/6edc116f08884985932592d0c060d470 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/6edc116f08884985932592d0c060d470 2024-11-08T09:24:05,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/6edc116f08884985932592d0c060d470, entries=250, sequenceid=42, filesize=47.0 K 2024-11-08T09:24:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/ada18ebd465a414c9ae4025b6458e1cb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/ada18ebd465a414c9ae4025b6458e1cb 2024-11-08T09:24:05,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/ada18ebd465a414c9ae4025b6458e1cb, entries=150, sequenceid=42, filesize=11.7 K 2024-11-08T09:24:05,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c80bb41a27ba46e5b3c077fcc8ef3ff2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c80bb41a27ba46e5b3c077fcc8ef3ff2 2024-11-08T09:24:05,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c80bb41a27ba46e5b3c077fcc8ef3ff2, entries=150, sequenceid=42, filesize=11.7 K 2024-11-08T09:24:05,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 22a585182af10d799ec85bb33e624e65 in 512ms, sequenceid=42, compaction requested=false 2024-11-08T09:24:05,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:05,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-08T09:24:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:05,248 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:24:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b49913df9272462198034a3e52556ba3_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057844709/Put/seqid=0 2024-11-08T09:24:05,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742136_1312 (size=12154) 2024-11-08T09:24:05,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:05,264 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b49913df9272462198034a3e52556ba3_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b49913df9272462198034a3e52556ba3_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:05,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/60c3d27691b341c2996b17b4feead658, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:05,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/60c3d27691b341c2996b17b4feead658 is 175, key is test_row_0/A:col10/1731057844709/Put/seqid=0 2024-11-08T09:24:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742137_1313 (size=30955) 2024-11-08T09:24:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:05,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057905385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057905388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057905389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057905390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057905397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057905501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057905501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057905501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057905501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057905512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,669 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/60c3d27691b341c2996b17b4feead658 2024-11-08T09:24:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/16fd210d0dcc4867b54f383395a33f4c is 50, key is test_row_0/B:col10/1731057844709/Put/seqid=0 2024-11-08T09:24:05,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742138_1314 (size=12001) 2024-11-08T09:24:05,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057905711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057905711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057905711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057905712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057905722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057906021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057906021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057906021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057906031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057906031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,047 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T09:24:06,080 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/16fd210d0dcc4867b54f383395a33f4c 2024-11-08T09:24:06,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/fd28ef61e63c4f218201d6bc87a5a74a is 50, key is test_row_0/C:col10/1731057844709/Put/seqid=0 2024-11-08T09:24:06,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742139_1315 (size=12001) 2024-11-08T09:24:06,095 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/fd28ef61e63c4f218201d6bc87a5a74a 2024-11-08T09:24:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/60c3d27691b341c2996b17b4feead658 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/60c3d27691b341c2996b17b4feead658 2024-11-08T09:24:06,110 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/60c3d27691b341c2996b17b4feead658, entries=150, sequenceid=54, filesize=30.2 K 2024-11-08T09:24:06,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/16fd210d0dcc4867b54f383395a33f4c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/16fd210d0dcc4867b54f383395a33f4c 2024-11-08T09:24:06,117 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/16fd210d0dcc4867b54f383395a33f4c, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T09:24:06,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/fd28ef61e63c4f218201d6bc87a5a74a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/fd28ef61e63c4f218201d6bc87a5a74a 2024-11-08T09:24:06,124 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/fd28ef61e63c4f218201d6bc87a5a74a, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T09:24:06,125 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 22a585182af10d799ec85bb33e624e65 in 877ms, sequenceid=54, compaction requested=true 2024-11-08T09:24:06,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:06,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:06,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-08T09:24:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-08T09:24:06,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-08T09:24:06,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1110 sec 2024-11-08T09:24:06,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.1180 sec 2024-11-08T09:24:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T09:24:06,162 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:06,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-08T09:24:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-08T09:24:06,164 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:06,165 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:06,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-08T09:24:06,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-08T09:24:06,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:06,317 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:24:06,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f47725720f5549e89e47c332efd3ac6f_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057845388/Put/seqid=0 2024-11-08T09:24:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742140_1316 (size=12154) 2024-11-08T09:24:06,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:06,332 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f47725720f5549e89e47c332efd3ac6f_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f47725720f5549e89e47c332efd3ac6f_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:06,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/2f3f3c4fd4c24fb9ba9f30fd63690c56, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:06,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/2f3f3c4fd4c24fb9ba9f30fd63690c56 is 175, key is test_row_0/A:col10/1731057845388/Put/seqid=0 2024-11-08T09:24:06,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742141_1317 (size=30955) 2024-11-08T09:24:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-08T09:24:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:06,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:06,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057906548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057906550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057906551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057906552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057906553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057906661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057906671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057906671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057906671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057906671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,737 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/2f3f3c4fd4c24fb9ba9f30fd63690c56 2024-11-08T09:24:06,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/a7e5d219218747a7a23e1b99e7b29640 is 50, key is test_row_0/B:col10/1731057845388/Put/seqid=0 2024-11-08T09:24:06,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742142_1318 (size=12001) 2024-11-08T09:24:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-08T09:24:06,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057906881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057906881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057906881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057906881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057906882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,148 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/a7e5d219218747a7a23e1b99e7b29640 2024-11-08T09:24:07,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/37c42cd9e5294419813a76b71a29bb79 is 50, key is test_row_0/C:col10/1731057845388/Put/seqid=0 2024-11-08T09:24:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742143_1319 (size=12001) 2024-11-08T09:24:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057907191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057907191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057907191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057907201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057907201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-08T09:24:07,561 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/37c42cd9e5294419813a76b71a29bb79 2024-11-08T09:24:07,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/2f3f3c4fd4c24fb9ba9f30fd63690c56 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2f3f3c4fd4c24fb9ba9f30fd63690c56 2024-11-08T09:24:07,568 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2f3f3c4fd4c24fb9ba9f30fd63690c56, entries=150, sequenceid=78, filesize=30.2 K 2024-11-08T09:24:07,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/a7e5d219218747a7a23e1b99e7b29640 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a7e5d219218747a7a23e1b99e7b29640 2024-11-08T09:24:07,572 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a7e5d219218747a7a23e1b99e7b29640, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T09:24:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/37c42cd9e5294419813a76b71a29bb79 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/37c42cd9e5294419813a76b71a29bb79 2024-11-08T09:24:07,576 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/37c42cd9e5294419813a76b71a29bb79, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T09:24:07,577 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 22a585182af10d799ec85bb33e624e65 in 1260ms, sequenceid=78, compaction requested=true 2024-11-08T09:24:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-08T09:24:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-08T09:24:07,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-08T09:24:07,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-08T09:24:07,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.4180 sec 2024-11-08T09:24:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:07,705 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:07,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:24:07,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e67b18605d354986acb06ca3632edde7_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057846544/Put/seqid=0 2024-11-08T09:24:07,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742144_1320 (size=17034) 2024-11-08T09:24:07,718 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:07,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e67b18605d354986acb06ca3632edde7_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e67b18605d354986acb06ca3632edde7_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:07,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/e6e013ccb3ab4731aecf55dd710799fb, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:07,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/e6e013ccb3ab4731aecf55dd710799fb is 175, key is test_row_0/A:col10/1731057846544/Put/seqid=0 2024-11-08T09:24:07,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742145_1321 (size=48139) 2024-11-08T09:24:07,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057907739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057907740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057907741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057907746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057907747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057907852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057907861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057907861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057907861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057907861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057908062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057908071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057908071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057908071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057908071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/e6e013ccb3ab4731aecf55dd710799fb 2024-11-08T09:24:08,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/6060bff6d8ee47da966a88f0ad532f4c is 50, key is test_row_0/B:col10/1731057846544/Put/seqid=0 2024-11-08T09:24:08,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742146_1322 (size=12001) 2024-11-08T09:24:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-08T09:24:08,311 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:08,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-08T09:24:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T09:24:08,314 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:08,314 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:08,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:08,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057908382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057908382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057908391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057908391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057908391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T09:24:08,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T09:24:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:08,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:08,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/6060bff6d8ee47da966a88f0ad532f4c 2024-11-08T09:24:08,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/a0634b2d86844859bae03ea3c0ac883d is 50, key is test_row_0/C:col10/1731057846544/Put/seqid=0 2024-11-08T09:24:08,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742147_1323 (size=12001) 2024-11-08T09:24:08,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/a0634b2d86844859bae03ea3c0ac883d 2024-11-08T09:24:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/e6e013ccb3ab4731aecf55dd710799fb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e6e013ccb3ab4731aecf55dd710799fb 2024-11-08T09:24:08,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e6e013ccb3ab4731aecf55dd710799fb, entries=250, sequenceid=92, filesize=47.0 K 2024-11-08T09:24:08,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/6060bff6d8ee47da966a88f0ad532f4c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/6060bff6d8ee47da966a88f0ad532f4c 2024-11-08T09:24:08,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/6060bff6d8ee47da966a88f0ad532f4c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T09:24:08,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/a0634b2d86844859bae03ea3c0ac883d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/a0634b2d86844859bae03ea3c0ac883d 2024-11-08T09:24:08,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/a0634b2d86844859bae03ea3c0ac883d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T09:24:08,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 22a585182af10d799ec85bb33e624e65 in 863ms, sequenceid=92, compaction requested=true 2024-11-08T09:24:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:08,569 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T09:24:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:08,569 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T09:24:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:08,570 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 189143 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T09:24:08,570 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T09:24:08,570 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/A is initiating minor compaction (all files) 2024-11-08T09:24:08,570 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/B is initiating minor compaction (all files) 2024-11-08T09:24:08,570 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/A in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:08,570 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/B in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:08,570 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/63a986687f4a4856a6e726728a401e0d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/ada18ebd465a414c9ae4025b6458e1cb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/16fd210d0dcc4867b54f383395a33f4c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a7e5d219218747a7a23e1b99e7b29640, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/6060bff6d8ee47da966a88f0ad532f4c] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=58.6 K 2024-11-08T09:24:08,570 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/adede391982e45e2beb4825e20642ab5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/6edc116f08884985932592d0c060d470, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/60c3d27691b341c2996b17b4feead658, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2f3f3c4fd4c24fb9ba9f30fd63690c56, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e6e013ccb3ab4731aecf55dd710799fb] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=184.7 K 2024-11-08T09:24:08,571 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:08,571 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/adede391982e45e2beb4825e20642ab5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/6edc116f08884985932592d0c060d470, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/60c3d27691b341c2996b17b4feead658, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2f3f3c4fd4c24fb9ba9f30fd63690c56, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e6e013ccb3ab4731aecf55dd710799fb] 2024-11-08T09:24:08,571 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a986687f4a4856a6e726728a401e0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731057844020 2024-11-08T09:24:08,571 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting adede391982e45e2beb4825e20642ab5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731057844020 2024-11-08T09:24:08,571 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting ada18ebd465a414c9ae4025b6458e1cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731057844053 2024-11-08T09:24:08,571 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6edc116f08884985932592d0c060d470, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731057844049 2024-11-08T09:24:08,571 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 16fd210d0dcc4867b54f383395a33f4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731057844707 2024-11-08T09:24:08,572 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c3d27691b341c2996b17b4feead658, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731057844707 2024-11-08T09:24:08,572 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a7e5d219218747a7a23e1b99e7b29640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057845388 2024-11-08T09:24:08,572 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3f3c4fd4c24fb9ba9f30fd63690c56, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057845388 2024-11-08T09:24:08,572 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 6060bff6d8ee47da966a88f0ad532f4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057846544 2024-11-08T09:24:08,572 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e013ccb3ab4731aecf55dd710799fb, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057846544 2024-11-08T09:24:08,580 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:08,582 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#B#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:08,583 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/2e66929e9ccc404ca3a0916de1e206b6 is 50, key is test_row_0/B:col10/1731057846544/Put/seqid=0 2024-11-08T09:24:08,583 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108071a9ca7b8b343f19c213f11eb908226_22a585182af10d799ec85bb33e624e65 store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:08,586 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108071a9ca7b8b343f19c213f11eb908226_22a585182af10d799ec85bb33e624e65, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:08,586 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108071a9ca7b8b343f19c213f11eb908226_22a585182af10d799ec85bb33e624e65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:08,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742148_1324 (size=12173) 2024-11-08T09:24:08,609 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/2e66929e9ccc404ca3a0916de1e206b6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2e66929e9ccc404ca3a0916de1e206b6 2024-11-08T09:24:08,618 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 22a585182af10d799ec85bb33e624e65/B of 22a585182af10d799ec85bb33e624e65 into 2e66929e9ccc404ca3a0916de1e206b6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:08,619 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:08,619 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/B, priority=11, startTime=1731057848569; duration=0sec 2024-11-08T09:24:08,619 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:08,619 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:B 2024-11-08T09:24:08,619 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T09:24:08,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T09:24:08,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:08,619 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T09:24:08,623 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T09:24:08,623 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/C is initiating minor compaction (all files) 2024-11-08T09:24:08,623 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/C in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:08,623 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/261112dc5623407a98949654f4bf899e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c80bb41a27ba46e5b3c077fcc8ef3ff2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/fd28ef61e63c4f218201d6bc87a5a74a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/37c42cd9e5294419813a76b71a29bb79, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/a0634b2d86844859bae03ea3c0ac883d] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=58.6 K 2024-11-08T09:24:08,626 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 261112dc5623407a98949654f4bf899e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731057844020 2024-11-08T09:24:08,626 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting c80bb41a27ba46e5b3c077fcc8ef3ff2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731057844053 2024-11-08T09:24:08,628 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting fd28ef61e63c4f218201d6bc87a5a74a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731057844707 2024-11-08T09:24:08,629 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 37c42cd9e5294419813a76b71a29bb79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057845388 2024-11-08T09:24:08,629 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a0634b2d86844859bae03ea3c0ac883d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057846544 2024-11-08T09:24:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110870f91e696dc8471cbfee74cba8b219fd_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057847746/Put/seqid=0 2024-11-08T09:24:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T09:24:08,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742149_1325 (size=4469) 2024-11-08T09:24:08,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742150_1326 (size=12154) 2024-11-08T09:24:08,653 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:08,654 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/7258bd8c14e04571b7291720c62344f2 is 50, key is test_row_0/C:col10/1731057846544/Put/seqid=0 2024-11-08T09:24:08,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:08,660 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110870f91e696dc8471cbfee74cba8b219fd_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110870f91e696dc8471cbfee74cba8b219fd_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:08,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/25775770ef30438a85b4a38e3aedb78b, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:08,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/25775770ef30438a85b4a38e3aedb78b is 175, key is test_row_0/A:col10/1731057847746/Put/seqid=0 2024-11-08T09:24:08,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742151_1327 (size=12173) 2024-11-08T09:24:08,673 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/7258bd8c14e04571b7291720c62344f2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/7258bd8c14e04571b7291720c62344f2 2024-11-08T09:24:08,681 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 22a585182af10d799ec85bb33e624e65/C of 22a585182af10d799ec85bb33e624e65 into 7258bd8c14e04571b7291720c62344f2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:08,681 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:08,681 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/C, priority=11, startTime=1731057848569; duration=0sec 2024-11-08T09:24:08,681 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:08,681 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:C 2024-11-08T09:24:08,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742152_1328 (size=30955) 2024-11-08T09:24:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:08,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:08,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057908916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057908917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057908917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057908922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057908923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T09:24:09,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057909031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057909031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057909031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057909031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057909041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,046 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#A#compaction#270 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:09,047 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/d2473c60dcf946d88e4d43e237d642b8 is 175, key is test_row_0/A:col10/1731057846544/Put/seqid=0 2024-11-08T09:24:09,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742153_1329 (size=31127) 2024-11-08T09:24:09,088 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/25775770ef30438a85b4a38e3aedb78b 2024-11-08T09:24:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/87f3936c80974064addda043f676107d is 50, key is test_row_0/B:col10/1731057847746/Put/seqid=0 2024-11-08T09:24:09,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742154_1330 (size=12001) 2024-11-08T09:24:09,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057909241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057909241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057909241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057909241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057909251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T09:24:09,454 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/d2473c60dcf946d88e4d43e237d642b8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d2473c60dcf946d88e4d43e237d642b8 2024-11-08T09:24:09,458 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 22a585182af10d799ec85bb33e624e65/A of 22a585182af10d799ec85bb33e624e65 into d2473c60dcf946d88e4d43e237d642b8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:09,458 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:09,458 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/A, priority=11, startTime=1731057848569; duration=0sec 2024-11-08T09:24:09,458 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:09,458 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:A 2024-11-08T09:24:09,498 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/87f3936c80974064addda043f676107d 2024-11-08T09:24:09,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/f65b65a7cca14480870cdba3500e2bba is 50, key is test_row_0/C:col10/1731057847746/Put/seqid=0 2024-11-08T09:24:09,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742155_1331 (size=12001) 2024-11-08T09:24:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057909551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057909551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057909561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057909561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057909561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:09,908 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/f65b65a7cca14480870cdba3500e2bba 2024-11-08T09:24:09,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/25775770ef30438a85b4a38e3aedb78b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/25775770ef30438a85b4a38e3aedb78b 2024-11-08T09:24:09,916 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/25775770ef30438a85b4a38e3aedb78b, entries=150, sequenceid=115, filesize=30.2 K 2024-11-08T09:24:09,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/87f3936c80974064addda043f676107d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/87f3936c80974064addda043f676107d 2024-11-08T09:24:09,920 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/87f3936c80974064addda043f676107d, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T09:24:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/f65b65a7cca14480870cdba3500e2bba as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/f65b65a7cca14480870cdba3500e2bba 2024-11-08T09:24:09,923 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/f65b65a7cca14480870cdba3500e2bba, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T09:24:09,925 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 22a585182af10d799ec85bb33e624e65 in 1306ms, sequenceid=115, compaction requested=false 2024-11-08T09:24:09,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:09,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:09,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-08T09:24:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-08T09:24:09,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-08T09:24:09,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-11-08T09:24:09,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.6160 sec 2024-11-08T09:24:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:10,067 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:10,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T09:24:10,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fd8c8ae7bb2b4428b8d5b3be659e1841_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057850065/Put/seqid=0 2024-11-08T09:24:10,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742156_1332 (size=17184) 2024-11-08T09:24:10,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057910099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057910099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057910100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057910101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057910101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057910211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057910211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057910211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057910211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057910221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057910421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057910421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057910421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057910421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057910431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T09:24:10,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:10,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-08T09:24:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T09:24:10,464 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:10,464 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:10,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:10,478 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:10,481 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fd8c8ae7bb2b4428b8d5b3be659e1841_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd8c8ae7bb2b4428b8d5b3be659e1841_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:10,482 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0efe1211d88243cc9c2ba42342622dee, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:10,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0efe1211d88243cc9c2ba42342622dee is 175, key is test_row_0/A:col10/1731057850065/Put/seqid=0 2024-11-08T09:24:10,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742157_1333 (size=48289) 2024-11-08T09:24:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T09:24:10,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:10,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057910732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057910732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057910732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057910732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057910741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:10,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:10,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:10,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T09:24:10,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0efe1211d88243cc9c2ba42342622dee 2024-11-08T09:24:10,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/a46f3207e3ac4bcfb5af5243a272a324 is 50, key is test_row_0/B:col10/1731057850065/Put/seqid=0 2024-11-08T09:24:10,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742158_1334 (size=12151) 2024-11-08T09:24:10,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:10,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:10,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:10,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:10,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T09:24:11,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057911241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:11,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057911241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:11,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057911241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057911252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:11,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057911261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:11,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/a46f3207e3ac4bcfb5af5243a272a324 2024-11-08T09:24:11,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/24e650d030a44946b7187e7d54c6da2b is 50, key is test_row_0/C:col10/1731057850065/Put/seqid=0 2024-11-08T09:24:11,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742159_1335 (size=12151) 2024-11-08T09:24:11,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:11,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:11,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T09:24:11,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:11,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:11,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:11,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/24e650d030a44946b7187e7d54c6da2b 2024-11-08T09:24:11,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0efe1211d88243cc9c2ba42342622dee as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0efe1211d88243cc9c2ba42342622dee 2024-11-08T09:24:11,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0efe1211d88243cc9c2ba42342622dee, entries=250, sequenceid=133, filesize=47.2 K 2024-11-08T09:24:11,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/a46f3207e3ac4bcfb5af5243a272a324 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a46f3207e3ac4bcfb5af5243a272a324 2024-11-08T09:24:11,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a46f3207e3ac4bcfb5af5243a272a324, entries=150, sequenceid=133, filesize=11.9 K 2024-11-08T09:24:11,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/24e650d030a44946b7187e7d54c6da2b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/24e650d030a44946b7187e7d54c6da2b 2024-11-08T09:24:11,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/24e650d030a44946b7187e7d54c6da2b, entries=150, sequenceid=133, filesize=11.9 K 2024-11-08T09:24:11,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 22a585182af10d799ec85bb33e624e65 in 1654ms, sequenceid=133, compaction requested=true 2024-11-08T09:24:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:11,721 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:11,721 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:11,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:11,722 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:11,722 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36325 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:11,722 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/A is initiating minor compaction (all files) 2024-11-08T09:24:11,722 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/B is initiating minor compaction (all files) 2024-11-08T09:24:11,722 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/A in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,722 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/B in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,722 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2e66929e9ccc404ca3a0916de1e206b6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/87f3936c80974064addda043f676107d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a46f3207e3ac4bcfb5af5243a272a324] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=35.5 K 2024-11-08T09:24:11,722 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d2473c60dcf946d88e4d43e237d642b8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/25775770ef30438a85b4a38e3aedb78b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0efe1211d88243cc9c2ba42342622dee] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=107.8 K 2024-11-08T09:24:11,723 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,723 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d2473c60dcf946d88e4d43e237d642b8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/25775770ef30438a85b4a38e3aedb78b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0efe1211d88243cc9c2ba42342622dee] 2024-11-08T09:24:11,723 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2473c60dcf946d88e4d43e237d642b8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057846544 2024-11-08T09:24:11,723 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e66929e9ccc404ca3a0916de1e206b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057846544 2024-11-08T09:24:11,723 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f3936c80974064addda043f676107d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731057847737 2024-11-08T09:24:11,723 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25775770ef30438a85b4a38e3aedb78b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731057847737 2024-11-08T09:24:11,723 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a46f3207e3ac4bcfb5af5243a272a324, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057848922 2024-11-08T09:24:11,723 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0efe1211d88243cc9c2ba42342622dee, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057848916 2024-11-08T09:24:11,730 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:11,731 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#B#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:11,731 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/5d4aa21980354b35beb486c59dc9263d is 50, key is test_row_0/B:col10/1731057850065/Put/seqid=0 2024-11-08T09:24:11,738 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108fb046e5856ac4e799dff034c3c525da5_22a585182af10d799ec85bb33e624e65 store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:11,740 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108fb046e5856ac4e799dff034c3c525da5_22a585182af10d799ec85bb33e624e65, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:11,740 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fb046e5856ac4e799dff034c3c525da5_22a585182af10d799ec85bb33e624e65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:11,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742160_1336 (size=12425) 2024-11-08T09:24:11,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742161_1337 (size=4469) 2024-11-08T09:24:11,747 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#A#compaction#279 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:11,747 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/5d4aa21980354b35beb486c59dc9263d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/5d4aa21980354b35beb486c59dc9263d 2024-11-08T09:24:11,748 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/2233cf19ed6346188ad344d8251fd2b9 is 175, key is test_row_0/A:col10/1731057850065/Put/seqid=0 2024-11-08T09:24:11,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742162_1338 (size=31379) 2024-11-08T09:24:11,754 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/B of 22a585182af10d799ec85bb33e624e65 into 5d4aa21980354b35beb486c59dc9263d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:11,754 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:11,754 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/B, priority=13, startTime=1731057851721; duration=0sec 2024-11-08T09:24:11,754 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:11,754 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:B 2024-11-08T09:24:11,755 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:11,755 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36325 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:11,755 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/C is initiating minor compaction (all files) 2024-11-08T09:24:11,756 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/C in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,756 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/7258bd8c14e04571b7291720c62344f2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/f65b65a7cca14480870cdba3500e2bba, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/24e650d030a44946b7187e7d54c6da2b] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=35.5 K 2024-11-08T09:24:11,756 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 7258bd8c14e04571b7291720c62344f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731057846544 2024-11-08T09:24:11,756 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f65b65a7cca14480870cdba3500e2bba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731057847737 2024-11-08T09:24:11,756 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e650d030a44946b7187e7d54c6da2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057848922 2024-11-08T09:24:11,758 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/2233cf19ed6346188ad344d8251fd2b9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2233cf19ed6346188ad344d8251fd2b9 2024-11-08T09:24:11,763 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/A of 22a585182af10d799ec85bb33e624e65 into 2233cf19ed6346188ad344d8251fd2b9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:11,763 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:11,763 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/A, priority=13, startTime=1731057851721; duration=0sec 2024-11-08T09:24:11,763 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:11,763 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:A 2024-11-08T09:24:11,764 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:11,764 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/5f1eb50d203b45ddb1f39e84b6772887 is 50, key is test_row_0/C:col10/1731057850065/Put/seqid=0 2024-11-08T09:24:11,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742163_1339 (size=12425) 2024-11-08T09:24:11,778 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/5f1eb50d203b45ddb1f39e84b6772887 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/5f1eb50d203b45ddb1f39e84b6772887 2024-11-08T09:24:11,786 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/C of 22a585182af10d799ec85bb33e624e65 into 5f1eb50d203b45ddb1f39e84b6772887(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:11,786 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:11,786 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/C, priority=13, startTime=1731057851721; duration=0sec 2024-11-08T09:24:11,786 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:11,786 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:C 2024-11-08T09:24:11,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T09:24:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:11,837 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T09:24:11,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a03deda4bb00400fa55c648929e3ee77_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057850097/Put/seqid=0 2024-11-08T09:24:11,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742164_1340 (size=12304) 2024-11-08T09:24:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:12,252 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a03deda4bb00400fa55c648929e3ee77_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a03deda4bb00400fa55c648929e3ee77_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4cf47e6879184a738460f485056498eb, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:12,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4cf47e6879184a738460f485056498eb is 175, key is test_row_0/A:col10/1731057850097/Put/seqid=0 2024-11-08T09:24:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:12,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:12,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742165_1341 (size=31105) 2024-11-08T09:24:12,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057912280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057912282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057912285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057912285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,289 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057912289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057912392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057912402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057912402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057912402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057912601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T09:24:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057912612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057912612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057912612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,657 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4cf47e6879184a738460f485056498eb 2024-11-08T09:24:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/eb75ac170764496b8fcee636c8600dfd is 50, key is test_row_0/B:col10/1731057850097/Put/seqid=0 2024-11-08T09:24:12,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742166_1342 (size=12151) 2024-11-08T09:24:12,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057912911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057912921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057912922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057912922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:13,067 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/eb75ac170764496b8fcee636c8600dfd 2024-11-08T09:24:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/da4f252ad0ce4d07a1b05701122f91cb is 50, key is test_row_0/C:col10/1731057850097/Put/seqid=0 2024-11-08T09:24:13,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742167_1343 (size=12151) 2024-11-08T09:24:13,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057913432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:13,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057913432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057913432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057913432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:13,482 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/da4f252ad0ce4d07a1b05701122f91cb 2024-11-08T09:24:13,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4cf47e6879184a738460f485056498eb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4cf47e6879184a738460f485056498eb 2024-11-08T09:24:13,488 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4cf47e6879184a738460f485056498eb, entries=150, sequenceid=156, filesize=30.4 K 2024-11-08T09:24:13,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/eb75ac170764496b8fcee636c8600dfd as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/eb75ac170764496b8fcee636c8600dfd 2024-11-08T09:24:13,493 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/eb75ac170764496b8fcee636c8600dfd, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T09:24:13,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/da4f252ad0ce4d07a1b05701122f91cb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/da4f252ad0ce4d07a1b05701122f91cb 2024-11-08T09:24:13,497 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/da4f252ad0ce4d07a1b05701122f91cb, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T09:24:13,498 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 22a585182af10d799ec85bb33e624e65 in 1661ms, sequenceid=156, compaction requested=false 2024-11-08T09:24:13,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:13,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-08T09:24:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-08T09:24:13,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-08T09:24:13,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0360 sec 2024-11-08T09:24:13,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.0400 sec 2024-11-08T09:24:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:14,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:14,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T09:24:14,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108040b69989e4542e18d2c451fb4fb93ca_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057852278/Put/seqid=0 2024-11-08T09:24:14,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742168_1344 (size=14794) 2024-11-08T09:24:14,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057914423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057914441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057914441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:14,447 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:14,448 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057914452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:14,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057914452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:14,457 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:14,457 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:14,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057914541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T09:24:14,631 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:14,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-08T09:24:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T09:24:14,633 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:14,634 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:14,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T09:24:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057914751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:14,766 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:14,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108040b69989e4542e18d2c451fb4fb93ca_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108040b69989e4542e18d2c451fb4fb93ca_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:14,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/47059b2a0afa4bfda4500df0625cd4bb, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:14,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/47059b2a0afa4bfda4500df0625cd4bb is 175, key is test_row_0/A:col10/1731057852278/Put/seqid=0 2024-11-08T09:24:14,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742169_1345 (size=39749) 2024-11-08T09:24:14,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:14,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:14,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:14,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:14,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:14,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:14,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:14,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T09:24:15,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057915062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:15,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:15,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/47059b2a0afa4bfda4500df0625cd4bb 2024-11-08T09:24:15,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/75c58f071dc046b7b9504cfe5aaf4735 is 50, key is test_row_0/B:col10/1731057852278/Put/seqid=0 2024-11-08T09:24:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742170_1346 (size=12151) 2024-11-08T09:24:15,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:15,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:15,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T09:24:15,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:15,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:15,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057915571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:15,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/75c58f071dc046b7b9504cfe5aaf4735 2024-11-08T09:24:15,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/d242f9a87f4743838ee65a56635b10e6 is 50, key is test_row_0/C:col10/1731057852278/Put/seqid=0 2024-11-08T09:24:15,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742171_1347 (size=12151) 2024-11-08T09:24:15,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T09:24:15,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:15,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:15,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:15,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:15,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/d242f9a87f4743838ee65a56635b10e6 2024-11-08T09:24:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/47059b2a0afa4bfda4500df0625cd4bb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/47059b2a0afa4bfda4500df0625cd4bb 2024-11-08T09:24:16,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/47059b2a0afa4bfda4500df0625cd4bb, entries=200, sequenceid=173, filesize=38.8 K 2024-11-08T09:24:16,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/75c58f071dc046b7b9504cfe5aaf4735 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/75c58f071dc046b7b9504cfe5aaf4735 2024-11-08T09:24:16,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/75c58f071dc046b7b9504cfe5aaf4735, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T09:24:16,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:16,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/d242f9a87f4743838ee65a56635b10e6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d242f9a87f4743838ee65a56635b10e6 2024-11-08T09:24:16,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:16,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d242f9a87f4743838ee65a56635b10e6, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T09:24:16,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 22a585182af10d799ec85bb33e624e65 in 1706ms, sequenceid=173, compaction requested=true 2024-11-08T09:24:16,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:16,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:16,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:16,012 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:16,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:16,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:16,012 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:16,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:16,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:16,012 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:16,012 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:16,012 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/A is initiating minor compaction (all files) 2024-11-08T09:24:16,012 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/B is initiating minor compaction (all files) 2024-11-08T09:24:16,012 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/A in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:16,012 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/B in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:16,013 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2233cf19ed6346188ad344d8251fd2b9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4cf47e6879184a738460f485056498eb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/47059b2a0afa4bfda4500df0625cd4bb] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=99.8 K 2024-11-08T09:24:16,013 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/5d4aa21980354b35beb486c59dc9263d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/eb75ac170764496b8fcee636c8600dfd, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/75c58f071dc046b7b9504cfe5aaf4735] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=35.9 K 2024-11-08T09:24:16,013 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:16,013 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2233cf19ed6346188ad344d8251fd2b9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4cf47e6879184a738460f485056498eb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/47059b2a0afa4bfda4500df0625cd4bb] 2024-11-08T09:24:16,013 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4aa21980354b35beb486c59dc9263d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057848922 2024-11-08T09:24:16,013 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2233cf19ed6346188ad344d8251fd2b9, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057848922 2024-11-08T09:24:16,013 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting eb75ac170764496b8fcee636c8600dfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731057850097 2024-11-08T09:24:16,013 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cf47e6879184a738460f485056498eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731057850097 2024-11-08T09:24:16,013 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 75c58f071dc046b7b9504cfe5aaf4735, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731057852278 2024-11-08T09:24:16,013 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47059b2a0afa4bfda4500df0625cd4bb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731057852278 2024-11-08T09:24:16,021 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:16,022 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#B#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:16,022 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411086a5a7ccad33a40f38fd91ec91e316092_22a585182af10d799ec85bb33e624e65 store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:16,023 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/e0aeeeeb111b4c86a7d4604dcc8e7508 is 50, key is test_row_0/B:col10/1731057852278/Put/seqid=0 2024-11-08T09:24:16,024 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411086a5a7ccad33a40f38fd91ec91e316092_22a585182af10d799ec85bb33e624e65, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:16,025 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086a5a7ccad33a40f38fd91ec91e316092_22a585182af10d799ec85bb33e624e65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:16,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742172_1348 (size=12527) 2024-11-08T09:24:16,035 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/e0aeeeeb111b4c86a7d4604dcc8e7508 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e0aeeeeb111b4c86a7d4604dcc8e7508 2024-11-08T09:24:16,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742173_1349 (size=4469) 2024-11-08T09:24:16,040 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/B of 22a585182af10d799ec85bb33e624e65 into e0aeeeeb111b4c86a7d4604dcc8e7508(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:16,040 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:16,040 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/B, priority=13, startTime=1731057856011; duration=0sec 2024-11-08T09:24:16,040 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:16,040 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:B 2024-11-08T09:24:16,040 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:16,043 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:16,043 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/C is initiating minor compaction (all files) 2024-11-08T09:24:16,043 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/C in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:16,043 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/5f1eb50d203b45ddb1f39e84b6772887, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/da4f252ad0ce4d07a1b05701122f91cb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d242f9a87f4743838ee65a56635b10e6] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=35.9 K 2024-11-08T09:24:16,043 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f1eb50d203b45ddb1f39e84b6772887, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057848922 2024-11-08T09:24:16,043 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting da4f252ad0ce4d07a1b05701122f91cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731057850097 2024-11-08T09:24:16,044 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d242f9a87f4743838ee65a56635b10e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731057852278 2024-11-08T09:24:16,051 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#C#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:16,052 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/2a43119a2ed84a0290971ae90a661b67 is 50, key is test_row_0/C:col10/1731057852278/Put/seqid=0 2024-11-08T09:24:16,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742174_1350 (size=12527) 2024-11-08T09:24:16,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T09:24:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:16,160 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T09:24:16,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110813879b6e80fd4023b8809d8689fb0519_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057854362/Put/seqid=0 2024-11-08T09:24:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742175_1351 (size=12304) 2024-11-08T09:24:16,437 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#A#compaction#288 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:16,438 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/9eea433e4fe2463aa6dde2095d13f44f is 175, key is test_row_0/A:col10/1731057852278/Put/seqid=0 2024-11-08T09:24:16,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742176_1352 (size=31481) 2024-11-08T09:24:16,466 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/2a43119a2ed84a0290971ae90a661b67 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2a43119a2ed84a0290971ae90a661b67 2024-11-08T09:24:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:16,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:16,470 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/C of 22a585182af10d799ec85bb33e624e65 into 2a43119a2ed84a0290971ae90a661b67(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:16,470 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:16,470 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/C, priority=13, startTime=1731057856012; duration=0sec 2024-11-08T09:24:16,470 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:16,470 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:C 2024-11-08T09:24:16,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057916518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057916523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057916524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057916525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:16,573 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110813879b6e80fd4023b8809d8689fb0519_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110813879b6e80fd4023b8809d8689fb0519_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/32e08d974ced44a9a2ce0b99045f8b6e, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/32e08d974ced44a9a2ce0b99045f8b6e is 175, key is test_row_0/A:col10/1731057854362/Put/seqid=0 2024-11-08T09:24:16,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742177_1353 (size=31105) 2024-11-08T09:24:16,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057916591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,598 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057916632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057916632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057916641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057916641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T09:24:16,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057916842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,846 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/9eea433e4fe2463aa6dde2095d13f44f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/9eea433e4fe2463aa6dde2095d13f44f 2024-11-08T09:24:16,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057916842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,850 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/A of 22a585182af10d799ec85bb33e624e65 into 9eea433e4fe2463aa6dde2095d13f44f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:16,850 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:16,850 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/A, priority=13, startTime=1731057856011; duration=0sec 2024-11-08T09:24:16,850 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:16,850 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:A 2024-11-08T09:24:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057916851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057916851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:16,977 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/32e08d974ced44a9a2ce0b99045f8b6e 2024-11-08T09:24:16,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/2a586448b91c49eda2b1e11193d00060 is 50, key is test_row_0/B:col10/1731057854362/Put/seqid=0 2024-11-08T09:24:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742178_1354 (size=12151) 2024-11-08T09:24:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057917151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057917151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057917162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057917171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,392 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/2a586448b91c49eda2b1e11193d00060 2024-11-08T09:24:17,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/172a13355ce54fbfb3aa758c650fbc25 is 50, key is test_row_0/C:col10/1731057854362/Put/seqid=0 2024-11-08T09:24:17,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742179_1355 (size=12151) 2024-11-08T09:24:17,404 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/172a13355ce54fbfb3aa758c650fbc25 2024-11-08T09:24:17,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/32e08d974ced44a9a2ce0b99045f8b6e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/32e08d974ced44a9a2ce0b99045f8b6e 2024-11-08T09:24:17,412 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/32e08d974ced44a9a2ce0b99045f8b6e, entries=150, sequenceid=193, filesize=30.4 K 2024-11-08T09:24:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/2a586448b91c49eda2b1e11193d00060 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2a586448b91c49eda2b1e11193d00060 2024-11-08T09:24:17,415 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2a586448b91c49eda2b1e11193d00060, entries=150, sequenceid=193, filesize=11.9 K 2024-11-08T09:24:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/172a13355ce54fbfb3aa758c650fbc25 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/172a13355ce54fbfb3aa758c650fbc25 2024-11-08T09:24:17,421 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/172a13355ce54fbfb3aa758c650fbc25, entries=150, sequenceid=193, filesize=11.9 K 2024-11-08T09:24:17,422 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 22a585182af10d799ec85bb33e624e65 in 1263ms, sequenceid=193, compaction requested=false 2024-11-08T09:24:17,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:17,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:17,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-08T09:24:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-08T09:24:17,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-08T09:24:17,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7890 sec 2024-11-08T09:24:17,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7930 sec 2024-11-08T09:24:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:17,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:17,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T09:24:17,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b68d109ef4234d7599fe4b6462757f9d_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057857663/Put/seqid=0 2024-11-08T09:24:17,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742180_1356 (size=17284) 2024-11-08T09:24:17,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:17,685 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b68d109ef4234d7599fe4b6462757f9d_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b68d109ef4234d7599fe4b6462757f9d_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:17,686 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/586af62efa644e1b8e8bbc823ba2fca9, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:17,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/586af62efa644e1b8e8bbc823ba2fca9 is 175, key is test_row_0/A:col10/1731057857663/Put/seqid=0 2024-11-08T09:24:17,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742181_1357 (size=48389) 2024-11-08T09:24:17,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/586af62efa644e1b8e8bbc823ba2fca9 2024-11-08T09:24:17,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/fec9cb4553ca43d0bdc63b5cf5a5c92c is 50, key is test_row_0/B:col10/1731057857663/Put/seqid=0 2024-11-08T09:24:17,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742182_1358 (size=12151) 2024-11-08T09:24:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057917711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057917712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057917713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057917713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057917821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057917831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057917831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057917831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057918031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057918041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057918041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057918041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/fec9cb4553ca43d0bdc63b5cf5a5c92c 2024-11-08T09:24:18,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/053985bd81114ca28bd64d11232b966e is 50, key is test_row_0/C:col10/1731057857663/Put/seqid=0 2024-11-08T09:24:18,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742183_1359 (size=12151) 2024-11-08T09:24:18,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/053985bd81114ca28bd64d11232b966e 2024-11-08T09:24:18,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/586af62efa644e1b8e8bbc823ba2fca9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/586af62efa644e1b8e8bbc823ba2fca9 2024-11-08T09:24:18,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/586af62efa644e1b8e8bbc823ba2fca9, entries=250, sequenceid=214, filesize=47.3 K 2024-11-08T09:24:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/fec9cb4553ca43d0bdc63b5cf5a5c92c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/fec9cb4553ca43d0bdc63b5cf5a5c92c 2024-11-08T09:24:18,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/fec9cb4553ca43d0bdc63b5cf5a5c92c, entries=150, sequenceid=214, filesize=11.9 K 2024-11-08T09:24:18,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/053985bd81114ca28bd64d11232b966e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/053985bd81114ca28bd64d11232b966e 2024-11-08T09:24:18,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/053985bd81114ca28bd64d11232b966e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-08T09:24:18,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 22a585182af10d799ec85bb33e624e65 in 475ms, sequenceid=214, compaction requested=true 2024-11-08T09:24:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:18,139 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:18,139 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:18,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:18,140 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:18,140 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/A is initiating minor compaction (all files) 2024-11-08T09:24:18,140 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/A in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:18,140 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/9eea433e4fe2463aa6dde2095d13f44f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/32e08d974ced44a9a2ce0b99045f8b6e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/586af62efa644e1b8e8bbc823ba2fca9] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=108.4 K 2024-11-08T09:24:18,140 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:18,140 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/9eea433e4fe2463aa6dde2095d13f44f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/32e08d974ced44a9a2ce0b99045f8b6e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/586af62efa644e1b8e8bbc823ba2fca9] 2024-11-08T09:24:18,140 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:18,140 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/B is initiating minor compaction (all files) 2024-11-08T09:24:18,141 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/B in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:18,141 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e0aeeeeb111b4c86a7d4604dcc8e7508, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2a586448b91c49eda2b1e11193d00060, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/fec9cb4553ca43d0bdc63b5cf5a5c92c] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=36.0 K 2024-11-08T09:24:18,141 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eea433e4fe2463aa6dde2095d13f44f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731057852278 2024-11-08T09:24:18,141 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e0aeeeeb111b4c86a7d4604dcc8e7508, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731057852278 2024-11-08T09:24:18,141 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e08d974ced44a9a2ce0b99045f8b6e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731057854362 2024-11-08T09:24:18,142 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a586448b91c49eda2b1e11193d00060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731057854362 2024-11-08T09:24:18,142 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 586af62efa644e1b8e8bbc823ba2fca9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731057856510 2024-11-08T09:24:18,142 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting fec9cb4553ca43d0bdc63b5cf5a5c92c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731057856524 2024-11-08T09:24:18,150 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:18,152 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:18,153 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/2ee31625e5554b5098ca4bdbcdfb9baf is 50, key is test_row_0/B:col10/1731057857663/Put/seqid=0 2024-11-08T09:24:18,156 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411083b9c755c052a40efa1f431c1b0888f1a_22a585182af10d799ec85bb33e624e65 store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:18,158 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411083b9c755c052a40efa1f431c1b0888f1a_22a585182af10d799ec85bb33e624e65, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:18,158 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083b9c755c052a40efa1f431c1b0888f1a_22a585182af10d799ec85bb33e624e65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:18,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742184_1360 (size=12629) 2024-11-08T09:24:18,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742185_1361 (size=4469) 2024-11-08T09:24:18,195 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#A#compaction#296 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:18,195 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/e1d5d78f807a4ee6bac9a4313a7d01f5 is 175, key is test_row_0/A:col10/1731057857663/Put/seqid=0 2024-11-08T09:24:18,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742186_1362 (size=31583) 2024-11-08T09:24:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:18,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:18,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T09:24:18,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110868e888a8e66c4863a35158a339dbdcf6_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057858344/Put/seqid=0 2024-11-08T09:24:18,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742187_1363 (size=14794) 2024-11-08T09:24:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057918378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057918381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057918383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057918383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057918491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057918491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057918501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057918501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,574 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/2ee31625e5554b5098ca4bdbcdfb9baf as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2ee31625e5554b5098ca4bdbcdfb9baf 2024-11-08T09:24:18,578 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/B of 22a585182af10d799ec85bb33e624e65 into 2ee31625e5554b5098ca4bdbcdfb9baf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:18,578 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:18,578 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/B, priority=13, startTime=1731057858139; duration=0sec 2024-11-08T09:24:18,578 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:18,578 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:B 2024-11-08T09:24:18,578 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:18,579 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:18,579 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/C is initiating minor compaction (all files) 2024-11-08T09:24:18,579 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/C in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:18,579 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2a43119a2ed84a0290971ae90a661b67, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/172a13355ce54fbfb3aa758c650fbc25, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/053985bd81114ca28bd64d11232b966e] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=36.0 K 2024-11-08T09:24:18,579 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a43119a2ed84a0290971ae90a661b67, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731057852278 2024-11-08T09:24:18,579 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 172a13355ce54fbfb3aa758c650fbc25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731057854362 2024-11-08T09:24:18,580 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 053985bd81114ca28bd64d11232b966e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731057856524 2024-11-08T09:24:18,585 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:18,586 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/de47625215924de99457c7180713d12f is 50, key is test_row_0/C:col10/1731057857663/Put/seqid=0 2024-11-08T09:24:18,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742188_1364 (size=12629) 2024-11-08T09:24:18,608 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/e1d5d78f807a4ee6bac9a4313a7d01f5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e1d5d78f807a4ee6bac9a4313a7d01f5 2024-11-08T09:24:18,612 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/A of 22a585182af10d799ec85bb33e624e65 into e1d5d78f807a4ee6bac9a4313a7d01f5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:18,612 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:18,612 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/A, priority=13, startTime=1731057858139; duration=0sec 2024-11-08T09:24:18,612 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:18,612 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:A 2024-11-08T09:24:18,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057918612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,617 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057918701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057918701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057918711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057918712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:18,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:18,759 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110868e888a8e66c4863a35158a339dbdcf6_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110868e888a8e66c4863a35158a339dbdcf6_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:18,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/ee699e1b966442e692b5d3a63e93c37a, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:18,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/ee699e1b966442e692b5d3a63e93c37a is 175, key is test_row_0/A:col10/1731057858344/Put/seqid=0 2024-11-08T09:24:18,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742189_1365 (size=39749) 2024-11-08T09:24:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T09:24:18,792 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:18,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-08T09:24:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T09:24:18,794 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:18,794 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:18,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T09:24:18,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:18,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:18,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:18,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:18,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:18,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:18,993 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/de47625215924de99457c7180713d12f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/de47625215924de99457c7180713d12f 2024-11-08T09:24:18,996 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/C of 22a585182af10d799ec85bb33e624e65 into de47625215924de99457c7180713d12f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:18,996 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:18,996 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/C, priority=13, startTime=1731057858139; duration=0sec 2024-11-08T09:24:18,996 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:18,996 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:C 2024-11-08T09:24:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057919011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057919011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057919021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057919021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T09:24:19,167 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/ee699e1b966442e692b5d3a63e93c37a 2024-11-08T09:24:19,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/082395dc592b49e49a2a9e1d7662c7b3 is 50, key is test_row_0/B:col10/1731057858344/Put/seqid=0 2024-11-08T09:24:19,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742190_1366 (size=12151) 2024-11-08T09:24:19,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:19,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:19,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T09:24:19,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057919521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057919521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057919531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057919531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:19,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/082395dc592b49e49a2a9e1d7662c7b3 2024-11-08T09:24:19,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/d0112fccc3974530b06555db5338d9a7 is 50, key is test_row_0/C:col10/1731057858344/Put/seqid=0 2024-11-08T09:24:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742191_1367 (size=12151) 2024-11-08T09:24:19,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:19,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T09:24:19,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/d0112fccc3974530b06555db5338d9a7 2024-11-08T09:24:19,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/ee699e1b966442e692b5d3a63e93c37a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/ee699e1b966442e692b5d3a63e93c37a 2024-11-08T09:24:19,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/ee699e1b966442e692b5d3a63e93c37a, entries=200, sequenceid=232, filesize=38.8 K 2024-11-08T09:24:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/082395dc592b49e49a2a9e1d7662c7b3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/082395dc592b49e49a2a9e1d7662c7b3 2024-11-08T09:24:19,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/082395dc592b49e49a2a9e1d7662c7b3, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T09:24:19,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/d0112fccc3974530b06555db5338d9a7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d0112fccc3974530b06555db5338d9a7 2024-11-08T09:24:20,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d0112fccc3974530b06555db5338d9a7, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T09:24:20,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 22a585182af10d799ec85bb33e624e65 in 1657ms, sequenceid=232, compaction requested=false 2024-11-08T09:24:20,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:20,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T09:24:20,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:20,014 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T09:24:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084a13f8b8f86c48fdbb00435e1476aa0f_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057858382/Put/seqid=0 2024-11-08T09:24:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742192_1368 (size=12304) 2024-11-08T09:24:20,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:20,433 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084a13f8b8f86c48fdbb00435e1476aa0f_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084a13f8b8f86c48fdbb00435e1476aa0f_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:20,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0bc16526c79c42bab3337b343d5a429f, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:20,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0bc16526c79c42bab3337b343d5a429f is 175, key is test_row_0/A:col10/1731057858382/Put/seqid=0 2024-11-08T09:24:20,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742193_1369 (size=31105) 2024-11-08T09:24:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:20,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:20,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057920587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057920591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057920595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057920596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057920701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057920702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057920702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057920711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,840 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0bc16526c79c42bab3337b343d5a429f 2024-11-08T09:24:20,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/77319c55dc684e8c839c1a73b3e96ed6 is 50, key is test_row_0/B:col10/1731057858382/Put/seqid=0 2024-11-08T09:24:20,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742194_1370 (size=12151) 2024-11-08T09:24:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057920911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057920911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057920911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057920922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T09:24:21,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057921221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057921231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057921231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057921231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,249 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/77319c55dc684e8c839c1a73b3e96ed6 2024-11-08T09:24:21,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/b0d4021253ea49fea0f0603ff58828a6 is 50, key is test_row_0/C:col10/1731057858382/Put/seqid=0 2024-11-08T09:24:21,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742195_1371 (size=12151) 2024-11-08T09:24:21,660 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/b0d4021253ea49fea0f0603ff58828a6 2024-11-08T09:24:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0bc16526c79c42bab3337b343d5a429f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0bc16526c79c42bab3337b343d5a429f 2024-11-08T09:24:21,666 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0bc16526c79c42bab3337b343d5a429f, entries=150, sequenceid=253, filesize=30.4 K 2024-11-08T09:24:21,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/77319c55dc684e8c839c1a73b3e96ed6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/77319c55dc684e8c839c1a73b3e96ed6 2024-11-08T09:24:21,670 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/77319c55dc684e8c839c1a73b3e96ed6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-08T09:24:21,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/b0d4021253ea49fea0f0603ff58828a6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/b0d4021253ea49fea0f0603ff58828a6 2024-11-08T09:24:21,674 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/b0d4021253ea49fea0f0603ff58828a6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-08T09:24:21,675 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 22a585182af10d799ec85bb33e624e65 in 1660ms, sequenceid=253, compaction requested=true 2024-11-08T09:24:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-08T09:24:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-08T09:24:21,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-08T09:24:21,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8820 sec 2024-11-08T09:24:21,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.8860 sec 2024-11-08T09:24:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:21,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T09:24:21,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089a625bcfc34b45ed88d723e772ea4075_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057861745/Put/seqid=0 2024-11-08T09:24:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742196_1372 (size=14994) 2024-11-08T09:24:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057921782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057921783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057921784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057921785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057921902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057921902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057921902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057921902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057922111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057922112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057922121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057922121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:22,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089a625bcfc34b45ed88d723e772ea4075_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a625bcfc34b45ed88d723e772ea4075_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:22,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/cb1360a22a924a4e923f77076a2d53b4, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:22,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/cb1360a22a924a4e923f77076a2d53b4 is 175, key is test_row_0/A:col10/1731057861745/Put/seqid=0 2024-11-08T09:24:22,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742197_1373 (size=39949) 2024-11-08T09:24:22,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057922421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057922431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057922431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057922442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/cb1360a22a924a4e923f77076a2d53b4 2024-11-08T09:24:22,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/d0bbccdd8c4748b9b4f8c259b8884a0f is 50, key is test_row_0/B:col10/1731057861745/Put/seqid=0 2024-11-08T09:24:22,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742198_1374 (size=12301) 2024-11-08T09:24:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46432 deadline: 1731057922651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,659 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 22a585182af10d799ec85bb33e624e65 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057922931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057922941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057922941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057922952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T09:24:22,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:22,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-08T09:24:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T09:24:22,963 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:22,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:22,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:22,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/d0bbccdd8c4748b9b4f8c259b8884a0f 2024-11-08T09:24:22,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c53f980e2ed54aa89ae69460f6df2025 is 50, key is test_row_0/C:col10/1731057861745/Put/seqid=0 2024-11-08T09:24:22,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742199_1375 (size=12301) 2024-11-08T09:24:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T09:24:23,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T09:24:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:23,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T09:24:23,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T09:24:23,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c53f980e2ed54aa89ae69460f6df2025 2024-11-08T09:24:23,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/cb1360a22a924a4e923f77076a2d53b4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/cb1360a22a924a4e923f77076a2d53b4 2024-11-08T09:24:23,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/cb1360a22a924a4e923f77076a2d53b4, entries=200, sequenceid=272, filesize=39.0 K 2024-11-08T09:24:23,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/d0bbccdd8c4748b9b4f8c259b8884a0f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/d0bbccdd8c4748b9b4f8c259b8884a0f 2024-11-08T09:24:23,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/d0bbccdd8c4748b9b4f8c259b8884a0f, entries=150, sequenceid=272, filesize=12.0 K 2024-11-08T09:24:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c53f980e2ed54aa89ae69460f6df2025 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c53f980e2ed54aa89ae69460f6df2025 2024-11-08T09:24:23,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c53f980e2ed54aa89ae69460f6df2025, entries=150, sequenceid=272, filesize=12.0 K 2024-11-08T09:24:23,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 22a585182af10d799ec85bb33e624e65 in 1667ms, sequenceid=272, compaction requested=true 2024-11-08T09:24:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:23,415 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:23,415 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:23,416 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:23,416 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:23,416 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/A is initiating minor compaction (all files) 2024-11-08T09:24:23,416 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/B is initiating minor compaction (all files) 2024-11-08T09:24:23,416 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/B in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,416 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/A in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,416 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e1d5d78f807a4ee6bac9a4313a7d01f5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/ee699e1b966442e692b5d3a63e93c37a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0bc16526c79c42bab3337b343d5a429f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/cb1360a22a924a4e923f77076a2d53b4] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=139.0 K 2024-11-08T09:24:23,416 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2ee31625e5554b5098ca4bdbcdfb9baf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/082395dc592b49e49a2a9e1d7662c7b3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/77319c55dc684e8c839c1a73b3e96ed6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/d0bbccdd8c4748b9b4f8c259b8884a0f] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=48.1 K 2024-11-08T09:24:23,416 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,416 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e1d5d78f807a4ee6bac9a4313a7d01f5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/ee699e1b966442e692b5d3a63e93c37a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0bc16526c79c42bab3337b343d5a429f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/cb1360a22a924a4e923f77076a2d53b4] 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ee31625e5554b5098ca4bdbcdfb9baf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731057856524 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1d5d78f807a4ee6bac9a4313a7d01f5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731057856524 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 082395dc592b49e49a2a9e1d7662c7b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731057857710 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee699e1b966442e692b5d3a63e93c37a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731057857710 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 77319c55dc684e8c839c1a73b3e96ed6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731057858369 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc16526c79c42bab3337b343d5a429f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731057858369 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1360a22a924a4e923f77076a2d53b4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731057860590 2024-11-08T09:24:23,417 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d0bbccdd8c4748b9b4f8c259b8884a0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731057860590 2024-11-08T09:24:23,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T09:24:23,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,421 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T09:24:23,429 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:23,430 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/f75237f44fc1432c9023648873be2234 is 50, key is test_row_0/B:col10/1731057861745/Put/seqid=0 2024-11-08T09:24:23,432 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:23,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081b9183db4314495ba1ef3e344706c4fd_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057861781/Put/seqid=0 2024-11-08T09:24:23,442 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108df60a2f34d7449c1a8ce7e32af4175e7_22a585182af10d799ec85bb33e624e65 store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742200_1376 (size=12915) 2024-11-08T09:24:23,444 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108df60a2f34d7449c1a8ce7e32af4175e7_22a585182af10d799ec85bb33e624e65, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:23,445 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108df60a2f34d7449c1a8ce7e32af4175e7_22a585182af10d799ec85bb33e624e65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:23,454 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/f75237f44fc1432c9023648873be2234 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/f75237f44fc1432c9023648873be2234 2024-11-08T09:24:23,459 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22a585182af10d799ec85bb33e624e65/B of 22a585182af10d799ec85bb33e624e65 into f75237f44fc1432c9023648873be2234(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:23,459 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:23,460 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/B, priority=12, startTime=1731057863415; duration=0sec 2024-11-08T09:24:23,460 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:23,460 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:B 2024-11-08T09:24:23,460 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:23,461 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:23,461 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/C is initiating minor compaction (all files) 2024-11-08T09:24:23,461 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/C in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:23,461 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/de47625215924de99457c7180713d12f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d0112fccc3974530b06555db5338d9a7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/b0d4021253ea49fea0f0603ff58828a6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c53f980e2ed54aa89ae69460f6df2025] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=48.1 K 2024-11-08T09:24:23,461 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting de47625215924de99457c7180713d12f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731057856524 2024-11-08T09:24:23,461 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d0112fccc3974530b06555db5338d9a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731057857710 2024-11-08T09:24:23,462 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b0d4021253ea49fea0f0603ff58828a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731057858369 2024-11-08T09:24:23,462 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting c53f980e2ed54aa89ae69460f6df2025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731057860590 2024-11-08T09:24:23,479 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:23,479 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/09be5100430c4aaf8df737797d95b4d9 is 50, key is test_row_0/C:col10/1731057861745/Put/seqid=0 2024-11-08T09:24:23,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742202_1378 (size=4469) 2024-11-08T09:24:23,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742201_1377 (size=12454) 2024-11-08T09:24:23,484 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#A#compaction#309 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:23,485 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/14a03770c3334933bbeedd357f24b4b9 is 175, key is test_row_0/A:col10/1731057861745/Put/seqid=0 2024-11-08T09:24:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:23,490 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081b9183db4314495ba1ef3e344706c4fd_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081b9183db4314495ba1ef3e344706c4fd_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:23,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0a8b79f87f134500840f231de286b8fe, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0a8b79f87f134500840f231de286b8fe is 175, key is test_row_0/A:col10/1731057861781/Put/seqid=0 2024-11-08T09:24:23,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742203_1379 (size=12915) 2024-11-08T09:24:23,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742204_1380 (size=31869) 2024-11-08T09:24:23,507 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/14a03770c3334933bbeedd357f24b4b9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/14a03770c3334933bbeedd357f24b4b9 2024-11-08T09:24:23,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742205_1381 (size=31255) 2024-11-08T09:24:23,513 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0a8b79f87f134500840f231de286b8fe 2024-11-08T09:24:23,514 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22a585182af10d799ec85bb33e624e65/A of 22a585182af10d799ec85bb33e624e65 into 14a03770c3334933bbeedd357f24b4b9(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:23,514 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:23,514 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/A, priority=12, startTime=1731057863415; duration=0sec 2024-11-08T09:24:23,515 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:23,515 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:A 2024-11-08T09:24:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/4fd4cf26a5da449d843c308815e6d4cb is 50, key is test_row_0/B:col10/1731057861781/Put/seqid=0 2024-11-08T09:24:23,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742206_1382 (size=12301) 2024-11-08T09:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T09:24:23,898 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/09be5100430c4aaf8df737797d95b4d9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/09be5100430c4aaf8df737797d95b4d9 2024-11-08T09:24:23,903 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22a585182af10d799ec85bb33e624e65/C of 22a585182af10d799ec85bb33e624e65 into 09be5100430c4aaf8df737797d95b4d9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:23,903 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:23,903 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/C, priority=12, startTime=1731057863415; duration=0sec 2024-11-08T09:24:23,903 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:23,903 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:C 2024-11-08T09:24:23,927 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/4fd4cf26a5da449d843c308815e6d4cb 2024-11-08T09:24:23,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/55c140c239c640b799626796e8520f58 is 50, key is test_row_0/C:col10/1731057861781/Put/seqid=0 2024-11-08T09:24:23,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742207_1383 (size=12301) 2024-11-08T09:24:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:23,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. as already flushing 2024-11-08T09:24:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057923981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057923988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057923990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057923990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,015 INFO [Thread-1416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1416. 2024-11-08T09:24:24,015 DEBUG [Thread-1416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,015 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,015 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,017 INFO [Thread-1414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1414. 2024-11-08T09:24:24,017 DEBUG [Thread-1414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,017 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,017 INFO [Thread-1420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1420. 2024-11-08T09:24:24,017 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,017 DEBUG [Thread-1420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,017 DEBUG [Thread-1420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,017 DEBUG [Thread-1420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,022 INFO [Thread-1412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1412. 2024-11-08T09:24:24,022 DEBUG [Thread-1412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,023 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,023 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,023 INFO [Thread-1418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1418. 2024-11-08T09:24:24,023 DEBUG [Thread-1418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,023 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,023 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T09:24:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057924102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057924102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057924102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057924102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46348 deadline: 1731057924311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46378 deadline: 1731057924311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46400 deadline: 1731057924312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46376 deadline: 1731057924312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:24,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:24:24,339 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/55c140c239c640b799626796e8520f58 2024-11-08T09:24:24,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/0a8b79f87f134500840f231de286b8fe as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0a8b79f87f134500840f231de286b8fe 2024-11-08T09:24:24,347 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0a8b79f87f134500840f231de286b8fe, entries=150, sequenceid=289, filesize=30.5 K 2024-11-08T09:24:24,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/4fd4cf26a5da449d843c308815e6d4cb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/4fd4cf26a5da449d843c308815e6d4cb 2024-11-08T09:24:24,351 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/4fd4cf26a5da449d843c308815e6d4cb, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T09:24:24,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/55c140c239c640b799626796e8520f58 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/55c140c239c640b799626796e8520f58 2024-11-08T09:24:24,355 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/55c140c239c640b799626796e8520f58, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T09:24:24,356 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 22a585182af10d799ec85bb33e624e65 in 934ms, sequenceid=289, compaction requested=false 2024-11-08T09:24:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-08T09:24:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-08T09:24:24,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-08T09:24:24,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3930 sec 2024-11-08T09:24:24,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.3970 sec 2024-11-08T09:24:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:24,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:24,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T09:24:24,624 INFO [Thread-1407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1407. 2024-11-08T09:24:24,624 INFO [Thread-1401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1401. 2024-11-08T09:24:24,624 INFO [Thread-1405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1405. 2024-11-08T09:24:24,624 DEBUG [Thread-1405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,624 DEBUG [Thread-1407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,624 DEBUG [Thread-1401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,624 DEBUG [Thread-1405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,624 DEBUG [Thread-1407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,624 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,624 DEBUG [Thread-1405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,624 DEBUG [Thread-1407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,624 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,625 INFO [Thread-1409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1409. 2024-11-08T09:24:24,625 DEBUG [Thread-1409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:24,626 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,626 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:24,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:24,653 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T09:24:24,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110873416db9931d4060bf707d468bb12c97_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057863988/Put/seqid=0 2024-11-08T09:24:24,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742208_1384 (size=12454) 2024-11-08T09:24:25,082 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:25,088 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110873416db9931d4060bf707d468bb12c97_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110873416db9931d4060bf707d468bb12c97_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:25,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/d7d90be41b9743aea4cd7929bdc8ca5b, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:25,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/d7d90be41b9743aea4cd7929bdc8ca5b is 175, key is test_row_0/A:col10/1731057863988/Put/seqid=0 2024-11-08T09:24:25,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742209_1385 (size=31255) 2024-11-08T09:24:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T09:24:25,112 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:25,497 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/d7d90be41b9743aea4cd7929bdc8ca5b 2024-11-08T09:24:25,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/e55ee8a9ce2e4192a86b8b0e9353449b is 50, key is test_row_0/B:col10/1731057863988/Put/seqid=0 2024-11-08T09:24:25,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742210_1386 (size=12301) 2024-11-08T09:24:25,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/e55ee8a9ce2e4192a86b8b0e9353449b 2024-11-08T09:24:25,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/2ff55de720924fa9ad3f5e7ce76ad116 is 50, key is test_row_0/C:col10/1731057863988/Put/seqid=0 2024-11-08T09:24:25,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742211_1387 (size=12301) 2024-11-08T09:24:26,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/2ff55de720924fa9ad3f5e7ce76ad116 2024-11-08T09:24:26,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/d7d90be41b9743aea4cd7929bdc8ca5b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d7d90be41b9743aea4cd7929bdc8ca5b 2024-11-08T09:24:26,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d7d90be41b9743aea4cd7929bdc8ca5b, entries=150, sequenceid=315, filesize=30.5 K 2024-11-08T09:24:26,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/e55ee8a9ce2e4192a86b8b0e9353449b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e55ee8a9ce2e4192a86b8b0e9353449b 2024-11-08T09:24:26,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e55ee8a9ce2e4192a86b8b0e9353449b, entries=150, sequenceid=315, filesize=12.0 K 2024-11-08T09:24:26,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/2ff55de720924fa9ad3f5e7ce76ad116 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2ff55de720924fa9ad3f5e7ce76ad116 2024-11-08T09:24:26,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2ff55de720924fa9ad3f5e7ce76ad116, entries=150, sequenceid=315, filesize=12.0 K 2024-11-08T09:24:26,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=6.71 KB/6870 for 22a585182af10d799ec85bb33e624e65 in 1728ms, sequenceid=315, compaction requested=true 2024-11-08T09:24:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:26,352 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:26,352 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22a585182af10d799ec85bb33e624e65:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/A is initiating minor compaction (all files) 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/B is initiating minor compaction (all files) 2024-11-08T09:24:26,353 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/A in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:26,353 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/B in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:26,353 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/14a03770c3334933bbeedd357f24b4b9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0a8b79f87f134500840f231de286b8fe, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d7d90be41b9743aea4cd7929bdc8ca5b] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=92.2 K 2024-11-08T09:24:26,353 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/f75237f44fc1432c9023648873be2234, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/4fd4cf26a5da449d843c308815e6d4cb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e55ee8a9ce2e4192a86b8b0e9353449b] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=36.6 K 2024-11-08T09:24:26,353 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/14a03770c3334933bbeedd357f24b4b9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0a8b79f87f134500840f231de286b8fe, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d7d90be41b9743aea4cd7929bdc8ca5b] 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f75237f44fc1432c9023648873be2234, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731057860590 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a03770c3334933bbeedd357f24b4b9, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731057860590 2024-11-08T09:24:26,353 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd4cf26a5da449d843c308815e6d4cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057861769 2024-11-08T09:24:26,354 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a8b79f87f134500840f231de286b8fe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057861769 2024-11-08T09:24:26,354 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e55ee8a9ce2e4192a86b8b0e9353449b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731057863988 2024-11-08T09:24:26,354 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d90be41b9743aea4cd7929bdc8ca5b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731057863988 2024-11-08T09:24:26,359 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:26,361 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:26,361 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108fc575f7b23254a2da3f0b13e26dfd681_22a585182af10d799ec85bb33e624e65 store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:26,361 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/c0bcd8057f9149c688b944b7fb439a76 is 50, key is test_row_0/B:col10/1731057863988/Put/seqid=0 2024-11-08T09:24:26,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742212_1388 (size=13017) 2024-11-08T09:24:26,369 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108fc575f7b23254a2da3f0b13e26dfd681_22a585182af10d799ec85bb33e624e65, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:26,369 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fc575f7b23254a2da3f0b13e26dfd681_22a585182af10d799ec85bb33e624e65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:26,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742213_1389 (size=4469) 2024-11-08T09:24:26,775 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:26,776 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/28ebff2f70eb49d78593711df98791dd is 175, key is test_row_0/A:col10/1731057863988/Put/seqid=0 2024-11-08T09:24:26,779 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/c0bcd8057f9149c688b944b7fb439a76 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/c0bcd8057f9149c688b944b7fb439a76 2024-11-08T09:24:26,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742214_1390 (size=31971) 2024-11-08T09:24:26,785 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/B of 22a585182af10d799ec85bb33e624e65 into c0bcd8057f9149c688b944b7fb439a76(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:26,785 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:26,785 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/B, priority=13, startTime=1731057866352; duration=0sec 2024-11-08T09:24:26,785 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:26,785 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:B 2024-11-08T09:24:26,785 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:26,786 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:26,786 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 22a585182af10d799ec85bb33e624e65/C is initiating minor compaction (all files) 2024-11-08T09:24:26,786 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22a585182af10d799ec85bb33e624e65/C in TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:26,786 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/09be5100430c4aaf8df737797d95b4d9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/55c140c239c640b799626796e8520f58, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2ff55de720924fa9ad3f5e7ce76ad116] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp, totalSize=36.6 K 2024-11-08T09:24:26,786 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 09be5100430c4aaf8df737797d95b4d9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731057860590 2024-11-08T09:24:26,787 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 55c140c239c640b799626796e8520f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731057861769 2024-11-08T09:24:26,787 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff55de720924fa9ad3f5e7ce76ad116, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731057863988 2024-11-08T09:24:26,794 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22a585182af10d799ec85bb33e624e65#C#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:26,794 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c5a8a5200e0d458ea7f421bd89e249b9 is 50, key is test_row_0/C:col10/1731057863988/Put/seqid=0 2024-11-08T09:24:26,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742215_1391 (size=13017) 2024-11-08T09:24:27,187 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/28ebff2f70eb49d78593711df98791dd as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/28ebff2f70eb49d78593711df98791dd 2024-11-08T09:24:27,191 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/A of 22a585182af10d799ec85bb33e624e65 into 28ebff2f70eb49d78593711df98791dd(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:27,191 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:27,191 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/A, priority=13, startTime=1731057866352; duration=0sec 2024-11-08T09:24:27,191 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:27,191 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:A 2024-11-08T09:24:27,202 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/c5a8a5200e0d458ea7f421bd89e249b9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c5a8a5200e0d458ea7f421bd89e249b9 2024-11-08T09:24:27,206 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22a585182af10d799ec85bb33e624e65/C of 22a585182af10d799ec85bb33e624e65 into c5a8a5200e0d458ea7f421bd89e249b9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:27,206 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22a585182af10d799ec85bb33e624e65: 2024-11-08T09:24:27,206 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., storeName=22a585182af10d799ec85bb33e624e65/C, priority=13, startTime=1731057866352; duration=0sec 2024-11-08T09:24:27,206 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:27,206 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22a585182af10d799ec85bb33e624e65:C 2024-11-08T09:24:29,613 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/adede391982e45e2beb4825e20642ab5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/6edc116f08884985932592d0c060d470, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/60c3d27691b341c2996b17b4feead658, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2f3f3c4fd4c24fb9ba9f30fd63690c56, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e6e013ccb3ab4731aecf55dd710799fb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d2473c60dcf946d88e4d43e237d642b8, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/25775770ef30438a85b4a38e3aedb78b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0efe1211d88243cc9c2ba42342622dee, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2233cf19ed6346188ad344d8251fd2b9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4cf47e6879184a738460f485056498eb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/47059b2a0afa4bfda4500df0625cd4bb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/9eea433e4fe2463aa6dde2095d13f44f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/32e08d974ced44a9a2ce0b99045f8b6e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/586af62efa644e1b8e8bbc823ba2fca9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e1d5d78f807a4ee6bac9a4313a7d01f5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/ee699e1b966442e692b5d3a63e93c37a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0bc16526c79c42bab3337b343d5a429f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/cb1360a22a924a4e923f77076a2d53b4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/14a03770c3334933bbeedd357f24b4b9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0a8b79f87f134500840f231de286b8fe, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d7d90be41b9743aea4cd7929bdc8ca5b] to archive 2024-11-08T09:24:29,614 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:24:29,616 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/adede391982e45e2beb4825e20642ab5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/adede391982e45e2beb4825e20642ab5 2024-11-08T09:24:29,617 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/6edc116f08884985932592d0c060d470 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/6edc116f08884985932592d0c060d470 2024-11-08T09:24:29,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/60c3d27691b341c2996b17b4feead658 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/60c3d27691b341c2996b17b4feead658 2024-11-08T09:24:29,620 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2f3f3c4fd4c24fb9ba9f30fd63690c56 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2f3f3c4fd4c24fb9ba9f30fd63690c56 2024-11-08T09:24:29,621 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e6e013ccb3ab4731aecf55dd710799fb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e6e013ccb3ab4731aecf55dd710799fb 2024-11-08T09:24:29,622 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d2473c60dcf946d88e4d43e237d642b8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d2473c60dcf946d88e4d43e237d642b8 2024-11-08T09:24:29,623 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/25775770ef30438a85b4a38e3aedb78b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/25775770ef30438a85b4a38e3aedb78b 2024-11-08T09:24:29,624 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0efe1211d88243cc9c2ba42342622dee to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0efe1211d88243cc9c2ba42342622dee 2024-11-08T09:24:29,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2233cf19ed6346188ad344d8251fd2b9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/2233cf19ed6346188ad344d8251fd2b9 2024-11-08T09:24:29,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4cf47e6879184a738460f485056498eb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4cf47e6879184a738460f485056498eb 2024-11-08T09:24:29,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/47059b2a0afa4bfda4500df0625cd4bb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/47059b2a0afa4bfda4500df0625cd4bb 2024-11-08T09:24:29,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/9eea433e4fe2463aa6dde2095d13f44f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/9eea433e4fe2463aa6dde2095d13f44f 2024-11-08T09:24:29,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/32e08d974ced44a9a2ce0b99045f8b6e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/32e08d974ced44a9a2ce0b99045f8b6e 2024-11-08T09:24:29,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/586af62efa644e1b8e8bbc823ba2fca9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/586af62efa644e1b8e8bbc823ba2fca9 2024-11-08T09:24:29,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e1d5d78f807a4ee6bac9a4313a7d01f5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/e1d5d78f807a4ee6bac9a4313a7d01f5 2024-11-08T09:24:29,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/ee699e1b966442e692b5d3a63e93c37a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/ee699e1b966442e692b5d3a63e93c37a 2024-11-08T09:24:29,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0bc16526c79c42bab3337b343d5a429f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0bc16526c79c42bab3337b343d5a429f 2024-11-08T09:24:29,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/cb1360a22a924a4e923f77076a2d53b4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/cb1360a22a924a4e923f77076a2d53b4 2024-11-08T09:24:29,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/14a03770c3334933bbeedd357f24b4b9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/14a03770c3334933bbeedd357f24b4b9 2024-11-08T09:24:29,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0a8b79f87f134500840f231de286b8fe to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/0a8b79f87f134500840f231de286b8fe 2024-11-08T09:24:29,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d7d90be41b9743aea4cd7929bdc8ca5b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/d7d90be41b9743aea4cd7929bdc8ca5b 2024-11-08T09:24:29,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/63a986687f4a4856a6e726728a401e0d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/ada18ebd465a414c9ae4025b6458e1cb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/16fd210d0dcc4867b54f383395a33f4c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a7e5d219218747a7a23e1b99e7b29640, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2e66929e9ccc404ca3a0916de1e206b6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/6060bff6d8ee47da966a88f0ad532f4c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/87f3936c80974064addda043f676107d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/5d4aa21980354b35beb486c59dc9263d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a46f3207e3ac4bcfb5af5243a272a324, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/eb75ac170764496b8fcee636c8600dfd, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e0aeeeeb111b4c86a7d4604dcc8e7508, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/75c58f071dc046b7b9504cfe5aaf4735, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2a586448b91c49eda2b1e11193d00060, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2ee31625e5554b5098ca4bdbcdfb9baf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/fec9cb4553ca43d0bdc63b5cf5a5c92c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/082395dc592b49e49a2a9e1d7662c7b3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/77319c55dc684e8c839c1a73b3e96ed6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/f75237f44fc1432c9023648873be2234, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/d0bbccdd8c4748b9b4f8c259b8884a0f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/4fd4cf26a5da449d843c308815e6d4cb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e55ee8a9ce2e4192a86b8b0e9353449b] to archive 2024-11-08T09:24:29,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:24:29,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/63a986687f4a4856a6e726728a401e0d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/63a986687f4a4856a6e726728a401e0d 2024-11-08T09:24:29,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/ada18ebd465a414c9ae4025b6458e1cb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/ada18ebd465a414c9ae4025b6458e1cb 2024-11-08T09:24:29,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/16fd210d0dcc4867b54f383395a33f4c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/16fd210d0dcc4867b54f383395a33f4c 2024-11-08T09:24:29,693 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a7e5d219218747a7a23e1b99e7b29640 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a7e5d219218747a7a23e1b99e7b29640 2024-11-08T09:24:29,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2e66929e9ccc404ca3a0916de1e206b6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2e66929e9ccc404ca3a0916de1e206b6 2024-11-08T09:24:29,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/6060bff6d8ee47da966a88f0ad532f4c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/6060bff6d8ee47da966a88f0ad532f4c 2024-11-08T09:24:29,696 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/87f3936c80974064addda043f676107d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/87f3936c80974064addda043f676107d 2024-11-08T09:24:29,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/5d4aa21980354b35beb486c59dc9263d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/5d4aa21980354b35beb486c59dc9263d 2024-11-08T09:24:29,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a46f3207e3ac4bcfb5af5243a272a324 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/a46f3207e3ac4bcfb5af5243a272a324 2024-11-08T09:24:29,700 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/eb75ac170764496b8fcee636c8600dfd to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/eb75ac170764496b8fcee636c8600dfd 2024-11-08T09:24:29,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e0aeeeeb111b4c86a7d4604dcc8e7508 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e0aeeeeb111b4c86a7d4604dcc8e7508 2024-11-08T09:24:29,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/75c58f071dc046b7b9504cfe5aaf4735 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/75c58f071dc046b7b9504cfe5aaf4735 2024-11-08T09:24:29,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2a586448b91c49eda2b1e11193d00060 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2a586448b91c49eda2b1e11193d00060 2024-11-08T09:24:29,705 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2ee31625e5554b5098ca4bdbcdfb9baf to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/2ee31625e5554b5098ca4bdbcdfb9baf 2024-11-08T09:24:29,706 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/fec9cb4553ca43d0bdc63b5cf5a5c92c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/fec9cb4553ca43d0bdc63b5cf5a5c92c 2024-11-08T09:24:29,707 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/082395dc592b49e49a2a9e1d7662c7b3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/082395dc592b49e49a2a9e1d7662c7b3 2024-11-08T09:24:29,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/77319c55dc684e8c839c1a73b3e96ed6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/77319c55dc684e8c839c1a73b3e96ed6 2024-11-08T09:24:29,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/f75237f44fc1432c9023648873be2234 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/f75237f44fc1432c9023648873be2234 2024-11-08T09:24:29,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/d0bbccdd8c4748b9b4f8c259b8884a0f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/d0bbccdd8c4748b9b4f8c259b8884a0f 2024-11-08T09:24:29,713 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/4fd4cf26a5da449d843c308815e6d4cb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/4fd4cf26a5da449d843c308815e6d4cb 2024-11-08T09:24:29,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e55ee8a9ce2e4192a86b8b0e9353449b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/e55ee8a9ce2e4192a86b8b0e9353449b 2024-11-08T09:24:29,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/261112dc5623407a98949654f4bf899e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c80bb41a27ba46e5b3c077fcc8ef3ff2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/fd28ef61e63c4f218201d6bc87a5a74a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/37c42cd9e5294419813a76b71a29bb79, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/7258bd8c14e04571b7291720c62344f2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/a0634b2d86844859bae03ea3c0ac883d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/f65b65a7cca14480870cdba3500e2bba, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/5f1eb50d203b45ddb1f39e84b6772887, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/24e650d030a44946b7187e7d54c6da2b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/da4f252ad0ce4d07a1b05701122f91cb, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2a43119a2ed84a0290971ae90a661b67, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d242f9a87f4743838ee65a56635b10e6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/172a13355ce54fbfb3aa758c650fbc25, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/de47625215924de99457c7180713d12f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/053985bd81114ca28bd64d11232b966e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d0112fccc3974530b06555db5338d9a7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/b0d4021253ea49fea0f0603ff58828a6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/09be5100430c4aaf8df737797d95b4d9, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c53f980e2ed54aa89ae69460f6df2025, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/55c140c239c640b799626796e8520f58, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2ff55de720924fa9ad3f5e7ce76ad116] to archive 2024-11-08T09:24:29,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:24:29,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/261112dc5623407a98949654f4bf899e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/261112dc5623407a98949654f4bf899e 2024-11-08T09:24:29,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c80bb41a27ba46e5b3c077fcc8ef3ff2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c80bb41a27ba46e5b3c077fcc8ef3ff2 2024-11-08T09:24:29,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/fd28ef61e63c4f218201d6bc87a5a74a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/fd28ef61e63c4f218201d6bc87a5a74a 2024-11-08T09:24:29,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/37c42cd9e5294419813a76b71a29bb79 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/37c42cd9e5294419813a76b71a29bb79 2024-11-08T09:24:29,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/7258bd8c14e04571b7291720c62344f2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/7258bd8c14e04571b7291720c62344f2 2024-11-08T09:24:29,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/a0634b2d86844859bae03ea3c0ac883d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/a0634b2d86844859bae03ea3c0ac883d 2024-11-08T09:24:29,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/f65b65a7cca14480870cdba3500e2bba to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/f65b65a7cca14480870cdba3500e2bba 2024-11-08T09:24:29,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/5f1eb50d203b45ddb1f39e84b6772887 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/5f1eb50d203b45ddb1f39e84b6772887 2024-11-08T09:24:29,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/24e650d030a44946b7187e7d54c6da2b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/24e650d030a44946b7187e7d54c6da2b 2024-11-08T09:24:29,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/da4f252ad0ce4d07a1b05701122f91cb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/da4f252ad0ce4d07a1b05701122f91cb 2024-11-08T09:24:29,734 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2a43119a2ed84a0290971ae90a661b67 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2a43119a2ed84a0290971ae90a661b67 2024-11-08T09:24:29,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d242f9a87f4743838ee65a56635b10e6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d242f9a87f4743838ee65a56635b10e6 2024-11-08T09:24:29,736 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/172a13355ce54fbfb3aa758c650fbc25 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/172a13355ce54fbfb3aa758c650fbc25 2024-11-08T09:24:29,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/de47625215924de99457c7180713d12f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/de47625215924de99457c7180713d12f 2024-11-08T09:24:29,738 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/053985bd81114ca28bd64d11232b966e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/053985bd81114ca28bd64d11232b966e 2024-11-08T09:24:29,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d0112fccc3974530b06555db5338d9a7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/d0112fccc3974530b06555db5338d9a7 2024-11-08T09:24:29,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/b0d4021253ea49fea0f0603ff58828a6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/b0d4021253ea49fea0f0603ff58828a6 2024-11-08T09:24:29,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/09be5100430c4aaf8df737797d95b4d9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/09be5100430c4aaf8df737797d95b4d9 2024-11-08T09:24:29,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c53f980e2ed54aa89ae69460f6df2025 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c53f980e2ed54aa89ae69460f6df2025 2024-11-08T09:24:29,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/55c140c239c640b799626796e8520f58 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/55c140c239c640b799626796e8520f58 2024-11-08T09:24:29,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/cf4d76213909:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2ff55de720924fa9ad3f5e7ce76ad116 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/2ff55de720924fa9ad3f5e7ce76ad116 2024-11-08T09:24:32,723 INFO [Thread-1403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1403. 2024-11-08T09:24:32,724 DEBUG [Thread-1403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:32,724 DEBUG [Thread-1403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:32,724 DEBUG [Thread-1403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:32,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1551 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4653 rows 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1550 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4650 rows 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1555 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4663 rows 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1558 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4674 rows 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1559 2024-11-08T09:24:32,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4674 rows 2024-11-08T09:24:32,724 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:24:32,724 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:24:32,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:32,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:32,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:32,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T09:24:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T09:24:32,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057872728"}]},"ts":"1731057872728"} 2024-11-08T09:24:32,729 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T09:24:32,729 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T09:24:32,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:24:32,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, UNASSIGN}] 2024-11-08T09:24:32,732 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, UNASSIGN 2024-11-08T09:24:32,733 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:32,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, UNASSIGN because future has completed 2024-11-08T09:24:32,734 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:24:32,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:24:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T09:24:32,886 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:32,887 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:24:32,887 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing 22a585182af10d799ec85bb33e624e65, disabling compactions & flushes 2024-11-08T09:24:32,887 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:32,887 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:32,887 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. after waiting 0 ms 2024-11-08T09:24:32,887 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:32,887 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing 22a585182af10d799ec85bb33e624e65 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-08T09:24:32,893 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c3cf7120f6004ab4b65551e57d7c6a9f_22a585182af10d799ec85bb33e624e65 is 50, key is test_row_0/A:col10/1731057864625/Put/seqid=0 2024-11-08T09:24:32,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742216_1392 (size=9914) 2024-11-08T09:24:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T09:24:33,300 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:33,307 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c3cf7120f6004ab4b65551e57d7c6a9f_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c3cf7120f6004ab4b65551e57d7c6a9f_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:33,308 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4c862bfb09ef4b3fbfeff3466974d704, store: [table=TestAcidGuarantees family=A region=22a585182af10d799ec85bb33e624e65] 2024-11-08T09:24:33,309 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4c862bfb09ef4b3fbfeff3466974d704 is 175, key is test_row_0/A:col10/1731057864625/Put/seqid=0 2024-11-08T09:24:33,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742217_1393 (size=22561) 2024-11-08T09:24:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T09:24:33,715 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4c862bfb09ef4b3fbfeff3466974d704 2024-11-08T09:24:33,728 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/adfec766e5c0422994a2994da258c11f is 50, key is test_row_0/B:col10/1731057864625/Put/seqid=0 2024-11-08T09:24:33,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742218_1394 (size=9857) 2024-11-08T09:24:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T09:24:34,133 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/adfec766e5c0422994a2994da258c11f 2024-11-08T09:24:34,144 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/068ebc9eb10d4941ac352391968963b1 is 50, key is test_row_0/C:col10/1731057864625/Put/seqid=0 2024-11-08T09:24:34,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742219_1395 (size=9857) 2024-11-08T09:24:34,550 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/068ebc9eb10d4941ac352391968963b1 2024-11-08T09:24:34,560 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/A/4c862bfb09ef4b3fbfeff3466974d704 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4c862bfb09ef4b3fbfeff3466974d704 2024-11-08T09:24:34,565 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4c862bfb09ef4b3fbfeff3466974d704, entries=100, sequenceid=323, filesize=22.0 K 2024-11-08T09:24:34,566 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/B/adfec766e5c0422994a2994da258c11f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/adfec766e5c0422994a2994da258c11f 2024-11-08T09:24:34,569 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/adfec766e5c0422994a2994da258c11f, entries=100, sequenceid=323, filesize=9.6 K 2024-11-08T09:24:34,569 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/.tmp/C/068ebc9eb10d4941ac352391968963b1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/068ebc9eb10d4941ac352391968963b1 2024-11-08T09:24:34,572 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/068ebc9eb10d4941ac352391968963b1, entries=100, sequenceid=323, filesize=9.6 K 2024-11-08T09:24:34,573 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 22a585182af10d799ec85bb33e624e65 in 1686ms, sequenceid=323, compaction requested=false 2024-11-08T09:24:34,577 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-11-08T09:24:34,577 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. 2024-11-08T09:24:34,578 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for 22a585182af10d799ec85bb33e624e65: Waiting for close lock at 1731057872887Running coprocessor pre-close hooks at 1731057872887Disabling compacts and flushes for region at 1731057872887Disabling writes for close at 1731057872887Obtaining lock to block concurrent updates at 1731057872887Preparing flush snapshotting stores in 22a585182af10d799ec85bb33e624e65 at 1731057872887Finished memstore snapshotting TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731057872887Flushing stores of TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65. at 1731057872888 (+1 ms)Flushing 22a585182af10d799ec85bb33e624e65/A: creating writer at 1731057872888Flushing mob file 22a585182af10d799ec85bb33e624e65/A: appending metadata at 1731057872893 (+5 ms)Flushing mob file 22a585182af10d799ec85bb33e624e65/A: closing flushed file at 1731057872893Flushing 22a585182af10d799ec85bb33e624e65/A: appending metadata at 1731057873308 (+415 ms)Flushing 22a585182af10d799ec85bb33e624e65/A: closing flushed file at 1731057873308Flushing 22a585182af10d799ec85bb33e624e65/B: creating writer at 1731057873724 (+416 ms)Flushing 22a585182af10d799ec85bb33e624e65/B: appending metadata at 1731057873728 (+4 ms)Flushing 22a585182af10d799ec85bb33e624e65/B: closing flushed file at 1731057873728Flushing 22a585182af10d799ec85bb33e624e65/C: creating writer at 1731057874141 (+413 ms)Flushing 22a585182af10d799ec85bb33e624e65/C: appending metadata at 1731057874144 (+3 ms)Flushing 22a585182af10d799ec85bb33e624e65/C: closing flushed file at 1731057874144Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8559024: reopening flushed file at 1731057874559 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fc0a22: reopening flushed file at 1731057874566 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23d66738: reopening flushed file at 1731057874569 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 22a585182af10d799ec85bb33e624e65 in 1686ms, sequenceid=323, compaction requested=false at 1731057874573 (+4 ms)Writing region close event to WAL at 1731057874574 (+1 ms)Running coprocessor post-close hooks at 1731057874577 (+3 ms)Closed at 1731057874577 2024-11-08T09:24:34,580 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed 22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,580 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=22a585182af10d799ec85bb33e624e65, regionState=CLOSED 2024-11-08T09:24:34,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:24:34,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-08T09:24:34,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure 22a585182af10d799ec85bb33e624e65, server=cf4d76213909,39151,1731057746690 in 1.8480 sec 2024-11-08T09:24:34,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-08T09:24:34,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22a585182af10d799ec85bb33e624e65, UNASSIGN in 1.8520 sec 2024-11-08T09:24:34,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-08T09:24:34,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8560 sec 2024-11-08T09:24:34,588 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057874588"}]},"ts":"1731057874588"} 2024-11-08T09:24:34,589 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T09:24:34,589 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T09:24:34,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8650 sec 2024-11-08T09:24:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T09:24:34,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:34,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T09:24:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:34,926 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T09:24:34,928 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:34,934 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,939 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/recovered.edits] 2024-11-08T09:24:34,943 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/28ebff2f70eb49d78593711df98791dd to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/28ebff2f70eb49d78593711df98791dd 2024-11-08T09:24:34,945 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4c862bfb09ef4b3fbfeff3466974d704 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/A/4c862bfb09ef4b3fbfeff3466974d704 2024-11-08T09:24:34,948 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/adfec766e5c0422994a2994da258c11f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/adfec766e5c0422994a2994da258c11f 2024-11-08T09:24:34,950 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/c0bcd8057f9149c688b944b7fb439a76 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/B/c0bcd8057f9149c688b944b7fb439a76 2024-11-08T09:24:34,953 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/068ebc9eb10d4941ac352391968963b1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/068ebc9eb10d4941ac352391968963b1 2024-11-08T09:24:34,955 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c5a8a5200e0d458ea7f421bd89e249b9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/C/c5a8a5200e0d458ea7f421bd89e249b9 2024-11-08T09:24:34,959 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/recovered.edits/326.seqid to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65/recovered.edits/326.seqid 2024-11-08T09:24:34,960 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,960 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T09:24:34,960 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T09:24:34,961 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T09:24:34,963 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108040b69989e4542e18d2c451fb4fb93ca_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108040b69989e4542e18d2c451fb4fb93ca_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,964 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110813879b6e80fd4023b8809d8689fb0519_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110813879b6e80fd4023b8809d8689fb0519_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,965 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081b9183db4314495ba1ef3e344706c4fd_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081b9183db4314495ba1ef3e344706c4fd_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,966 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084a13f8b8f86c48fdbb00435e1476aa0f_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084a13f8b8f86c48fdbb00435e1476aa0f_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,967 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110868e888a8e66c4863a35158a339dbdcf6_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110868e888a8e66c4863a35158a339dbdcf6_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,968 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086ff0e71b9ee94cc699e6adc3926ddb42_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086ff0e71b9ee94cc699e6adc3926ddb42_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,969 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110870f91e696dc8471cbfee74cba8b219fd_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110870f91e696dc8471cbfee74cba8b219fd_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,969 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110873416db9931d4060bf707d468bb12c97_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110873416db9931d4060bf707d468bb12c97_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,970 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a625bcfc34b45ed88d723e772ea4075_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a625bcfc34b45ed88d723e772ea4075_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,971 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a03deda4bb00400fa55c648929e3ee77_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a03deda4bb00400fa55c648929e3ee77_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,971 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b49913df9272462198034a3e52556ba3_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b49913df9272462198034a3e52556ba3_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,972 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b68d109ef4234d7599fe4b6462757f9d_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b68d109ef4234d7599fe4b6462757f9d_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,973 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c3cf7120f6004ab4b65551e57d7c6a9f_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c3cf7120f6004ab4b65551e57d7c6a9f_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,974 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e67b18605d354986acb06ca3632edde7_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e67b18605d354986acb06ca3632edde7_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,975 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f2b40d0282f542a4b8f1cdd6c402dbec_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f2b40d0282f542a4b8f1cdd6c402dbec_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,975 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f47725720f5549e89e47c332efd3ac6f_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f47725720f5549e89e47c332efd3ac6f_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,976 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd8c8ae7bb2b4428b8d5b3be659e1841_22a585182af10d799ec85bb33e624e65 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd8c8ae7bb2b4428b8d5b3be659e1841_22a585182af10d799ec85bb33e624e65 2024-11-08T09:24:34,977 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T09:24:34,978 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:34,980 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T09:24:34,981 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T09:24:34,982 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:34,982 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T09:24:34,983 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731057874982"}]},"ts":"9223372036854775807"} 2024-11-08T09:24:34,984 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T09:24:34,984 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 22a585182af10d799ec85bb33e624e65, NAME => 'TestAcidGuarantees,,1731057840874.22a585182af10d799ec85bb33e624e65.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T09:24:34,984 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T09:24:34,984 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731057874984"}]},"ts":"9223372036854775807"} 2024-11-08T09:24:34,985 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T09:24:34,986 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:34,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-11-08T09:24:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T09:24:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T09:24:35,031 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:35,039 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 242), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=305 (was 347), ProcessCount=11 (was 11), AvailableMemoryMB=5457 (was 5499) 2024-11-08T09:24:35,049 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=305, ProcessCount=11, AvailableMemoryMB=5457 2024-11-08T09:24:35,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:24:35,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:24:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T09:24:35,053 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T09:24:35,053 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:35,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-08T09:24:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-08T09:24:35,055 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T09:24:35,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742220_1396 (size=959) 2024-11-08T09:24:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-08T09:24:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-08T09:24:35,465 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:24:35,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742221_1397 (size=53) 2024-11-08T09:24:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-08T09:24:35,876 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:24:35,876 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4c6f6f9fc6a2212b0e327dab2405a5f0, disabling compactions & flushes 2024-11-08T09:24:35,877 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:35,877 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:35,877 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. after waiting 0 ms 2024-11-08T09:24:35,877 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:35,877 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:35,877 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: Waiting for close lock at 1731057875876Disabling compacts and flushes for region at 1731057875876Disabling writes for close at 1731057875877 (+1 ms)Writing region close event to WAL at 1731057875877Closed at 1731057875877 2024-11-08T09:24:35,879 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T09:24:35,880 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731057875879"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731057875879"}]},"ts":"1731057875879"} 2024-11-08T09:24:35,883 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T09:24:35,884 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T09:24:35,885 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057875884"}]},"ts":"1731057875884"} 2024-11-08T09:24:35,886 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T09:24:35,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, ASSIGN}] 2024-11-08T09:24:35,888 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, ASSIGN 2024-11-08T09:24:35,889 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, ASSIGN; state=OFFLINE, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=false 2024-11-08T09:24:36,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=4c6f6f9fc6a2212b0e327dab2405a5f0, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:36,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, ASSIGN because future has completed 2024-11-08T09:24:36,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:24:36,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T09:24:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-08T09:24:36,202 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:36,202 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:24:36,203 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,203 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:24:36,203 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,203 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,205 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,207 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c6f6f9fc6a2212b0e327dab2405a5f0 columnFamilyName A 2024-11-08T09:24:36,208 DEBUG [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:36,208 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] regionserver.HStore(327): Store=4c6f6f9fc6a2212b0e327dab2405a5f0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:36,208 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,210 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c6f6f9fc6a2212b0e327dab2405a5f0 columnFamilyName B 2024-11-08T09:24:36,210 DEBUG [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:36,211 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] regionserver.HStore(327): Store=4c6f6f9fc6a2212b0e327dab2405a5f0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:36,211 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,213 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c6f6f9fc6a2212b0e327dab2405a5f0 columnFamilyName C 2024-11-08T09:24:36,213 DEBUG [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:24:36,214 INFO [StoreOpener-4c6f6f9fc6a2212b0e327dab2405a5f0-1 {}] regionserver.HStore(327): Store=4c6f6f9fc6a2212b0e327dab2405a5f0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:24:36,214 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,215 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,216 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,217 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,217 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,218 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:24:36,220 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,222 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:24:36,222 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 4c6f6f9fc6a2212b0e327dab2405a5f0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71909227, jitterRate=0.07153098285198212}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:24:36,222 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:36,222 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: Running coprocessor pre-open hook at 1731057876203Writing region info on filesystem at 1731057876203Initializing all the Stores at 1731057876205 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057876205Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057876205Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057876205Cleaning up temporary data from old regions at 1731057876217 (+12 ms)Running coprocessor post-open hooks at 1731057876222 (+5 ms)Region opened successfully at 1731057876222 2024-11-08T09:24:36,223 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., pid=110, masterSystemTime=1731057876196 2024-11-08T09:24:36,224 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:36,224 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:36,225 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=4c6f6f9fc6a2212b0e327dab2405a5f0, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:24:36,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:24:36,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-08T09:24:36,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 in 184 msec 2024-11-08T09:24:36,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-08T09:24:36,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, ASSIGN in 341 msec 2024-11-08T09:24:36,230 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T09:24:36,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057876230"}]},"ts":"1731057876230"} 2024-11-08T09:24:36,231 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T09:24:36,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T09:24:36,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-08T09:24:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-08T09:24:37,203 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:37,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5524d889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,208 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69da6c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,211 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d6ece1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,215 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T09:24:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,35807,1731057745600 2024-11-08T09:24:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40c80e3 2024-11-08T09:24:37,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T09:24:37,217 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T09:24:37,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc683cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,219 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28457d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,220 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df4086b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43681357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,222 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e5bc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b7d303, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39982088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,225 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65776518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,226 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8c6d12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4c85de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,227 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de819e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,228 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15073135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38404fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,230 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@715ee1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,231 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@122b7c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@125833c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,232 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d688130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e476b55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea16345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,235 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2918cdda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,236 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fca2683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2334b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e37429d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dae9113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1af8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,240 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1138e91b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,241 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6265b2c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c40e836, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:24:37,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:24:37,243 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:24:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:24:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:24:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ebcaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:24:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:24:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:37,244 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:24:37,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e0aa2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:24:37,245 DEBUG [Thread-1822 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,245 DEBUG [Thread-1813 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,245 DEBUG [Thread-1820 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,245 DEBUG [Thread-1807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,246 DEBUG [Thread-1811 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,246 DEBUG [Thread-1815 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,246 DEBUG [Thread-1824 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,247 DEBUG [Thread-1811 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,247 DEBUG [Thread-1811 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,247 DEBUG [Thread-1809 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,248 DEBUG [Thread-1818 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-08T09:24:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T09:24:37,251 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:37,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,252 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:37,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,254 DEBUG [Thread-1826 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:24:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:24:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:24:37,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:24:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2] 2024-11-08T09:24:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:37,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:37,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:24:37,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/3dc030da12b1480aa2b764f6ee50b161 is 50, key is test_row_1/A:col10/1731057877257/Put/seqid=0 2024-11-08T09:24:37,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057937291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057937298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057937299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057937300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057937300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742222_1398 (size=14337) 2024-11-08T09:24:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T09:24:37,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-08T09:24:37,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:37,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:37,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:37,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057937411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057937412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057937412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057937412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057937412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-08T09:24:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:37,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T09:24:37,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057937622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057937631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057937631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057937631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057937631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-08T09:24:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:37,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/3dc030da12b1480aa2b764f6ee50b161 2024-11-08T09:24:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:37,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/29b853c2c49643f0bed3aa5e89708556 is 50, key is test_row_1/B:col10/1731057877257/Put/seqid=0 2024-11-08T09:24:37,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742223_1399 (size=9657) 2024-11-08T09:24:37,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/29b853c2c49643f0bed3aa5e89708556 2024-11-08T09:24:37,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/d90edd9cdde24d348ca3810523d62ef6 is 50, key is test_row_1/C:col10/1731057877257/Put/seqid=0 2024-11-08T09:24:37,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742224_1400 (size=9657) 2024-11-08T09:24:37,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/d90edd9cdde24d348ca3810523d62ef6 2024-11-08T09:24:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/3dc030da12b1480aa2b764f6ee50b161 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3dc030da12b1480aa2b764f6ee50b161 2024-11-08T09:24:37,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3dc030da12b1480aa2b764f6ee50b161, entries=200, sequenceid=14, filesize=14.0 K 2024-11-08T09:24:37,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/29b853c2c49643f0bed3aa5e89708556 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/29b853c2c49643f0bed3aa5e89708556 2024-11-08T09:24:37,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/29b853c2c49643f0bed3aa5e89708556, entries=100, sequenceid=14, filesize=9.4 K 2024-11-08T09:24:37,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/d90edd9cdde24d348ca3810523d62ef6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d90edd9cdde24d348ca3810523d62ef6 2024-11-08T09:24:37,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d90edd9cdde24d348ca3810523d62ef6, entries=100, sequenceid=14, filesize=9.4 K 2024-11-08T09:24:37,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 530ms, sequenceid=14, compaction requested=false 2024-11-08T09:24:37,791 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T09:24:37,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:37,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-08T09:24:37,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:37,864 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:24:37,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/11578fc1246d49349e33c2b1d27a732c is 50, key is test_row_0/A:col10/1731057877299/Put/seqid=0 2024-11-08T09:24:37,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742225_1401 (size=12001) 2024-11-08T09:24:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T09:24:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:37,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057937945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057937946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057937947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057937947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:37,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057937948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057938051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057938061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057938061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057938062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057938062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057938261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,272 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/11578fc1246d49349e33c2b1d27a732c 2024-11-08T09:24:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057938271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057938272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057938272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057938272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/3d590d0f7fc74633a91a1997ceca0ee5 is 50, key is test_row_0/B:col10/1731057877299/Put/seqid=0 2024-11-08T09:24:38,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742226_1402 (size=12001) 2024-11-08T09:24:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T09:24:38,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057938571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057938581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057938581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057938582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057938582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:38,686 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/3d590d0f7fc74633a91a1997ceca0ee5 2024-11-08T09:24:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1248ea8096064d9195e3f89c457e6d32 is 50, key is test_row_0/C:col10/1731057877299/Put/seqid=0 2024-11-08T09:24:38,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742227_1403 (size=12001) 2024-11-08T09:24:39,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057939081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:39,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057939091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:39,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057939092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:39,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057939092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:39,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057939092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:39,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:39,101 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1248ea8096064d9195e3f89c457e6d32 2024-11-08T09:24:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/11578fc1246d49349e33c2b1d27a732c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/11578fc1246d49349e33c2b1d27a732c 2024-11-08T09:24:39,107 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/11578fc1246d49349e33c2b1d27a732c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T09:24:39,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/3d590d0f7fc74633a91a1997ceca0ee5 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/3d590d0f7fc74633a91a1997ceca0ee5 2024-11-08T09:24:39,110 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/3d590d0f7fc74633a91a1997ceca0ee5, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T09:24:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1248ea8096064d9195e3f89c457e6d32 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1248ea8096064d9195e3f89c457e6d32 2024-11-08T09:24:39,114 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1248ea8096064d9195e3f89c457e6d32, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T09:24:39,115 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1251ms, sequenceid=38, compaction requested=false 2024-11-08T09:24:39,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:39,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:39,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-08T09:24:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-08T09:24:39,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-08T09:24:39,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-11-08T09:24:39,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8720 sec 2024-11-08T09:24:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T09:24:39,401 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:39,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-08T09:24:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T09:24:39,403 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:39,404 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:39,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T09:24:39,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T09:24:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:39,556 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:24:39,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/b1ce16fa0c1e4905af88ea6a16499f7e is 50, key is test_row_0/A:col10/1731057877946/Put/seqid=0 2024-11-08T09:24:39,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742228_1404 (size=12001) 2024-11-08T09:24:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T09:24:39,963 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/b1ce16fa0c1e4905af88ea6a16499f7e 2024-11-08T09:24:40,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/70ac0e8752d049c5bae73a12de2452d4 is 50, key is test_row_0/B:col10/1731057877946/Put/seqid=0 2024-11-08T09:24:40,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742229_1405 (size=12001) 2024-11-08T09:24:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T09:24:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:40,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:40,101 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T09:24:40,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057940165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057940164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057940166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057940174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057940174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057940281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057940281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057940282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057940282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057940282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,429 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/70ac0e8752d049c5bae73a12de2452d4 2024-11-08T09:24:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4e501d8691474e7bb446fc9774405de2 is 50, key is test_row_0/C:col10/1731057877946/Put/seqid=0 2024-11-08T09:24:40,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742230_1406 (size=12001) 2024-11-08T09:24:40,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057940491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057940491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057940491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057940491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057940491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T09:24:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057940802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057940802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057940802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057940802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057940802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:40,842 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4e501d8691474e7bb446fc9774405de2 2024-11-08T09:24:40,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/b1ce16fa0c1e4905af88ea6a16499f7e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/b1ce16fa0c1e4905af88ea6a16499f7e 2024-11-08T09:24:40,848 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/b1ce16fa0c1e4905af88ea6a16499f7e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T09:24:40,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/70ac0e8752d049c5bae73a12de2452d4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/70ac0e8752d049c5bae73a12de2452d4 2024-11-08T09:24:40,855 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/70ac0e8752d049c5bae73a12de2452d4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T09:24:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4e501d8691474e7bb446fc9774405de2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4e501d8691474e7bb446fc9774405de2 2024-11-08T09:24:40,860 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4e501d8691474e7bb446fc9774405de2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T09:24:40,861 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1305ms, sequenceid=50, compaction requested=true 2024-11-08T09:24:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-08T09:24:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-08T09:24:40,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-08T09:24:40,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-11-08T09:24:40,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.4620 sec 2024-11-08T09:24:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:41,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T09:24:41,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/339c95bbc27344c09cb868467af7bfe2 is 50, key is test_row_0/A:col10/1731057881315/Put/seqid=0 2024-11-08T09:24:41,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742231_1407 (size=16681) 2024-11-08T09:24:41,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057941327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057941327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057941327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057941328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057941328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057941441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057941441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057941441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057941442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057941442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T09:24:41,552 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:41,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-08T09:24:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T09:24:41,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:41,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:41,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:41,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057941651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057941651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057941652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057941652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057941652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T09:24:41,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-08T09:24:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:41,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:41,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/339c95bbc27344c09cb868467af7bfe2 2024-11-08T09:24:41,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/02549750bb7740fe8355c73e3eb96d8f is 50, key is test_row_0/B:col10/1731057881315/Put/seqid=0 2024-11-08T09:24:41,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742232_1408 (size=12001) 2024-11-08T09:24:41,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-08T09:24:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:41,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T09:24:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057941961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057941961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057941962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057941962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057941962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:42,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-08T09:24:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/02549750bb7740fe8355c73e3eb96d8f 2024-11-08T09:24:42,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/f4907ffda6ba4900b47a1ba777d355cd is 50, key is test_row_0/C:col10/1731057881315/Put/seqid=0 2024-11-08T09:24:42,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742233_1409 (size=12001) 2024-11-08T09:24:42,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-08T09:24:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T09:24:42,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-08T09:24:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-08T09:24:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:42,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057942472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:42,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057942472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:42,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057942472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:42,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057942472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:42,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057942481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:42,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/f4907ffda6ba4900b47a1ba777d355cd 2024-11-08T09:24:42,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/339c95bbc27344c09cb868467af7bfe2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/339c95bbc27344c09cb868467af7bfe2 2024-11-08T09:24:42,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/339c95bbc27344c09cb868467af7bfe2, entries=250, sequenceid=77, filesize=16.3 K 2024-11-08T09:24:42,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/02549750bb7740fe8355c73e3eb96d8f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/02549750bb7740fe8355c73e3eb96d8f 2024-11-08T09:24:42,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/02549750bb7740fe8355c73e3eb96d8f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T09:24:42,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/f4907ffda6ba4900b47a1ba777d355cd as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/f4907ffda6ba4900b47a1ba777d355cd 2024-11-08T09:24:42,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/f4907ffda6ba4900b47a1ba777d355cd, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T09:24:42,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1244ms, sequenceid=77, compaction requested=true 2024-11-08T09:24:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:42,559 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:42,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:42,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:42,560 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:42,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:42,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:42,560 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:42,560 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/A is initiating minor compaction (all files) 2024-11-08T09:24:42,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:42,560 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/A in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,561 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3dc030da12b1480aa2b764f6ee50b161, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/11578fc1246d49349e33c2b1d27a732c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/b1ce16fa0c1e4905af88ea6a16499f7e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/339c95bbc27344c09cb868467af7bfe2] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=53.7 K 2024-11-08T09:24:42,561 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc030da12b1480aa2b764f6ee50b161, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731057877257 2024-11-08T09:24:42,561 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:42,562 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/B is initiating minor compaction (all files) 2024-11-08T09:24:42,562 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/B in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,562 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11578fc1246d49349e33c2b1d27a732c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731057877296 2024-11-08T09:24:42,562 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/29b853c2c49643f0bed3aa5e89708556, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/3d590d0f7fc74633a91a1997ceca0ee5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/70ac0e8752d049c5bae73a12de2452d4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/02549750bb7740fe8355c73e3eb96d8f] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=44.6 K 2024-11-08T09:24:42,562 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1ce16fa0c1e4905af88ea6a16499f7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731057877944 2024-11-08T09:24:42,562 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b853c2c49643f0bed3aa5e89708556, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731057877257 2024-11-08T09:24:42,562 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339c95bbc27344c09cb868467af7bfe2, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731057880162 2024-11-08T09:24:42,562 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d590d0f7fc74633a91a1997ceca0ee5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731057877296 2024-11-08T09:24:42,562 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ac0e8752d049c5bae73a12de2452d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731057877944 2024-11-08T09:24:42,563 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 02549750bb7740fe8355c73e3eb96d8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731057880162 2024-11-08T09:24:42,569 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:42,570 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#A#compaction#335 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:42,570 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/b3844fe184154e3299aac4f875ab3ad0 is 50, key is test_row_0/B:col10/1731057881315/Put/seqid=0 2024-11-08T09:24:42,570 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/fe7ec362ba2541949117b1b1f579bf54 is 50, key is test_row_0/A:col10/1731057881315/Put/seqid=0 2024-11-08T09:24:42,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742234_1410 (size=12139) 2024-11-08T09:24:42,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742235_1411 (size=12139) 2024-11-08T09:24:42,579 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/fe7ec362ba2541949117b1b1f579bf54 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/fe7ec362ba2541949117b1b1f579bf54 2024-11-08T09:24:42,582 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/b3844fe184154e3299aac4f875ab3ad0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3844fe184154e3299aac4f875ab3ad0 2024-11-08T09:24:42,584 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/A of 4c6f6f9fc6a2212b0e327dab2405a5f0 into fe7ec362ba2541949117b1b1f579bf54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:42,584 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:42,584 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/A, priority=12, startTime=1731057882559; duration=0sec 2024-11-08T09:24:42,584 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:42,585 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:A 2024-11-08T09:24:42,585 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:42,585 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:42,585 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/C is initiating minor compaction (all files) 2024-11-08T09:24:42,585 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/C in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,586 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d90edd9cdde24d348ca3810523d62ef6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1248ea8096064d9195e3f89c457e6d32, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4e501d8691474e7bb446fc9774405de2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/f4907ffda6ba4900b47a1ba777d355cd] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=44.6 K 2024-11-08T09:24:42,586 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90edd9cdde24d348ca3810523d62ef6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731057877257 2024-11-08T09:24:42,586 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1248ea8096064d9195e3f89c457e6d32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731057877296 2024-11-08T09:24:42,586 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e501d8691474e7bb446fc9774405de2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731057877944 2024-11-08T09:24:42,587 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4907ffda6ba4900b47a1ba777d355cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731057880162 2024-11-08T09:24:42,587 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/B of 4c6f6f9fc6a2212b0e327dab2405a5f0 into b3844fe184154e3299aac4f875ab3ad0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:42,587 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:42,587 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/B, priority=12, startTime=1731057882560; duration=0sec 2024-11-08T09:24:42,588 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:42,588 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:B 2024-11-08T09:24:42,595 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:42,595 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/faffac527aac478d89e999f9de03a06d is 50, key is test_row_0/C:col10/1731057881315/Put/seqid=0 2024-11-08T09:24:42,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742236_1412 (size=12139) 2024-11-08T09:24:42,604 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/faffac527aac478d89e999f9de03a06d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/faffac527aac478d89e999f9de03a06d 2024-11-08T09:24:42,608 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/C of 4c6f6f9fc6a2212b0e327dab2405a5f0 into faffac527aac478d89e999f9de03a06d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:42,608 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:42,608 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/C, priority=12, startTime=1731057882560; duration=0sec 2024-11-08T09:24:42,608 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:42,608 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:C 2024-11-08T09:24:42,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-08T09:24:42,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:42,622 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T09:24:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/32326d1094b74e7a9ce2389227965a8d is 50, key is test_row_0/A:col10/1731057881316/Put/seqid=0 2024-11-08T09:24:42,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742237_1413 (size=12001) 2024-11-08T09:24:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T09:24:43,030 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/32326d1094b74e7a9ce2389227965a8d 2024-11-08T09:24:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/fa5efcc4807943619b16c6d34539c26f is 50, key is test_row_0/B:col10/1731057881316/Put/seqid=0 2024-11-08T09:24:43,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742238_1414 (size=12001) 2024-11-08T09:24:43,041 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/fa5efcc4807943619b16c6d34539c26f 2024-11-08T09:24:43,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/5fab9d3ee0f1429d99fbc9f81f6f6b40 is 50, key is test_row_0/C:col10/1731057881316/Put/seqid=0 2024-11-08T09:24:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742239_1415 (size=12001) 2024-11-08T09:24:43,052 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/5fab9d3ee0f1429d99fbc9f81f6f6b40 2024-11-08T09:24:43,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/32326d1094b74e7a9ce2389227965a8d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/32326d1094b74e7a9ce2389227965a8d 2024-11-08T09:24:43,065 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/32326d1094b74e7a9ce2389227965a8d, entries=150, sequenceid=89, filesize=11.7 K 2024-11-08T09:24:43,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/fa5efcc4807943619b16c6d34539c26f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/fa5efcc4807943619b16c6d34539c26f 2024-11-08T09:24:43,070 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/fa5efcc4807943619b16c6d34539c26f, entries=150, sequenceid=89, filesize=11.7 K 2024-11-08T09:24:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/5fab9d3ee0f1429d99fbc9f81f6f6b40 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/5fab9d3ee0f1429d99fbc9f81f6f6b40 2024-11-08T09:24:43,076 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/5fab9d3ee0f1429d99fbc9f81f6f6b40, entries=150, sequenceid=89, filesize=11.7 K 2024-11-08T09:24:43,077 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 456ms, sequenceid=89, compaction requested=false 2024-11-08T09:24:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-08T09:24:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-08T09:24:43,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-08T09:24:43,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5240 sec 2024-11-08T09:24:43,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.5290 sec 2024-11-08T09:24:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:43,493 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:43,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:24:43,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/a44e6729b5cd4771974dd7213f09ec6a is 50, key is test_row_0/A:col10/1731057883491/Put/seqid=0 2024-11-08T09:24:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742240_1416 (size=9657) 2024-11-08T09:24:43,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057943525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057943530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057943531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057943532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057943533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057943641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057943641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057943641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057943652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057943652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T09:24:43,701 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:43,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-08T09:24:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T09:24:43,704 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:43,704 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:43,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T09:24:43,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T09:24:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:43,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:43,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:43,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057943851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057943851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057943851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057943862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057943862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:43,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/a44e6729b5cd4771974dd7213f09ec6a 2024-11-08T09:24:43,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/a4e2b029a33c4f2f9f437adb74714e76 is 50, key is test_row_0/B:col10/1731057883491/Put/seqid=0 2024-11-08T09:24:43,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742241_1417 (size=9657) 2024-11-08T09:24:44,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T09:24:44,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:44,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T09:24:44,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T09:24:44,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057944161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057944171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057944171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057944172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057944172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/a4e2b029a33c4f2f9f437adb74714e76 2024-11-08T09:24:44,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T09:24:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/a6c847e547bd490abe803bd52b95dbd0 is 50, key is test_row_0/C:col10/1731057883491/Put/seqid=0 2024-11-08T09:24:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742242_1418 (size=9657) 2024-11-08T09:24:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T09:24:44,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T09:24:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:44,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T09:24:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:44,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057944671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057944682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057944682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057944691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057944691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:44,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/a6c847e547bd490abe803bd52b95dbd0 2024-11-08T09:24:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/a44e6729b5cd4771974dd7213f09ec6a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/a44e6729b5cd4771974dd7213f09ec6a 2024-11-08T09:24:44,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/a44e6729b5cd4771974dd7213f09ec6a, entries=100, sequenceid=100, filesize=9.4 K 2024-11-08T09:24:44,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/a4e2b029a33c4f2f9f437adb74714e76 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/a4e2b029a33c4f2f9f437adb74714e76 2024-11-08T09:24:44,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/a4e2b029a33c4f2f9f437adb74714e76, entries=100, sequenceid=100, filesize=9.4 K 2024-11-08T09:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/a6c847e547bd490abe803bd52b95dbd0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/a6c847e547bd490abe803bd52b95dbd0 2024-11-08T09:24:44,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/a6c847e547bd490abe803bd52b95dbd0, entries=100, sequenceid=100, filesize=9.4 K 2024-11-08T09:24:44,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1249ms, sequenceid=100, compaction requested=true 2024-11-08T09:24:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:44,742 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:44,742 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/A is initiating minor compaction (all files) 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/B is initiating minor compaction (all files) 2024-11-08T09:24:44,743 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/A in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,743 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/B in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,743 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/fe7ec362ba2541949117b1b1f579bf54, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/32326d1094b74e7a9ce2389227965a8d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/a44e6729b5cd4771974dd7213f09ec6a] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=33.0 K 2024-11-08T09:24:44,743 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3844fe184154e3299aac4f875ab3ad0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/fa5efcc4807943619b16c6d34539c26f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/a4e2b029a33c4f2f9f437adb74714e76] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=33.0 K 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe7ec362ba2541949117b1b1f579bf54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731057880162 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b3844fe184154e3299aac4f875ab3ad0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731057880162 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting fa5efcc4807943619b16c6d34539c26f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731057881316 2024-11-08T09:24:44,743 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32326d1094b74e7a9ce2389227965a8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731057881316 2024-11-08T09:24:44,744 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a4e2b029a33c4f2f9f437adb74714e76, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731057883486 2024-11-08T09:24:44,744 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting a44e6729b5cd4771974dd7213f09ec6a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731057883486 2024-11-08T09:24:44,748 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#A#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:44,749 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/0cdc590af0974b54becaf6ffb26458e1 is 50, key is test_row_0/A:col10/1731057883491/Put/seqid=0 2024-11-08T09:24:44,751 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:44,751 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/82ff505e8ac7473db4506f826cb6fe46 is 50, key is test_row_0/B:col10/1731057883491/Put/seqid=0 2024-11-08T09:24:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742244_1420 (size=12241) 2024-11-08T09:24:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742243_1419 (size=12241) 2024-11-08T09:24:44,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T09:24:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:44,773 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:24:44,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/834e74b85ced43a3b99d4efba697e9c4 is 50, key is test_row_0/A:col10/1731057883532/Put/seqid=0 2024-11-08T09:24:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742245_1421 (size=12001) 2024-11-08T09:24:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T09:24:45,216 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/834e74b85ced43a3b99d4efba697e9c4 2024-11-08T09:24:45,219 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/82ff505e8ac7473db4506f826cb6fe46 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/82ff505e8ac7473db4506f826cb6fe46 2024-11-08T09:24:45,219 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/0cdc590af0974b54becaf6ffb26458e1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0cdc590af0974b54becaf6ffb26458e1 2024-11-08T09:24:45,225 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/B of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 82ff505e8ac7473db4506f826cb6fe46(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:45,225 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:45,225 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/B, priority=13, startTime=1731057884742; duration=0sec 2024-11-08T09:24:45,225 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:45,225 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:B 2024-11-08T09:24:45,225 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:45,225 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/A of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 0cdc590af0974b54becaf6ffb26458e1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:45,226 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:45,226 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/A, priority=13, startTime=1731057884742; duration=0sec 2024-11-08T09:24:45,226 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:45,226 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:A 2024-11-08T09:24:45,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/757907f2a66d4581803cdfd94da535c0 is 50, key is test_row_0/B:col10/1731057883532/Put/seqid=0 2024-11-08T09:24:45,228 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:45,228 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/C is initiating minor compaction (all files) 2024-11-08T09:24:45,228 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/C in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:45,229 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/faffac527aac478d89e999f9de03a06d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/5fab9d3ee0f1429d99fbc9f81f6f6b40, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/a6c847e547bd490abe803bd52b95dbd0] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=33.0 K 2024-11-08T09:24:45,229 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting faffac527aac478d89e999f9de03a06d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731057880162 2024-11-08T09:24:45,229 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fab9d3ee0f1429d99fbc9f81f6f6b40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731057881316 2024-11-08T09:24:45,229 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c847e547bd490abe803bd52b95dbd0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731057883486 2024-11-08T09:24:45,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742246_1422 (size=12001) 2024-11-08T09:24:45,232 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/757907f2a66d4581803cdfd94da535c0 2024-11-08T09:24:45,248 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#C#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:45,249 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/2d1664f195f543c5927dd2b8c54f32f7 is 50, key is test_row_0/C:col10/1731057883491/Put/seqid=0 2024-11-08T09:24:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/caf40f7566974de18ba23f3c2f8012a1 is 50, key is test_row_0/C:col10/1731057883532/Put/seqid=0 2024-11-08T09:24:45,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742247_1423 (size=12241) 2024-11-08T09:24:45,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742248_1424 (size=12001) 2024-11-08T09:24:45,666 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/2d1664f195f543c5927dd2b8c54f32f7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/2d1664f195f543c5927dd2b8c54f32f7 2024-11-08T09:24:45,669 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/caf40f7566974de18ba23f3c2f8012a1 2024-11-08T09:24:45,670 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/C of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 2d1664f195f543c5927dd2b8c54f32f7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:45,670 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:45,670 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/C, priority=13, startTime=1731057884742; duration=0sec 2024-11-08T09:24:45,670 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:45,670 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:C 2024-11-08T09:24:45,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/834e74b85ced43a3b99d4efba697e9c4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/834e74b85ced43a3b99d4efba697e9c4 2024-11-08T09:24:45,675 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/834e74b85ced43a3b99d4efba697e9c4, entries=150, sequenceid=125, filesize=11.7 K 2024-11-08T09:24:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/757907f2a66d4581803cdfd94da535c0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/757907f2a66d4581803cdfd94da535c0 2024-11-08T09:24:45,679 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/757907f2a66d4581803cdfd94da535c0, entries=150, sequenceid=125, filesize=11.7 K 2024-11-08T09:24:45,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/caf40f7566974de18ba23f3c2f8012a1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/caf40f7566974de18ba23f3c2f8012a1 2024-11-08T09:24:45,683 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/caf40f7566974de18ba23f3c2f8012a1, entries=150, sequenceid=125, filesize=11.7 K 2024-11-08T09:24:45,684 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 911ms, sequenceid=125, compaction requested=false 2024-11-08T09:24:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:45,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-08T09:24:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-08T09:24:45,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-08T09:24:45,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9810 sec 2024-11-08T09:24:45,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9850 sec 2024-11-08T09:24:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:45,707 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:45,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:24:45,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/29996e0011a04e15838548fe6f0dece6 is 50, key is test_row_0/A:col10/1731057885706/Put/seqid=0 2024-11-08T09:24:45,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742249_1425 (size=14541) 2024-11-08T09:24:45,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057945742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057945743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057945743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057945746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057945747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T09:24:45,851 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:45,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-08T09:24:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T09:24:45,853 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:45,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:45,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:45,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057945861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057945861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057945861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057945861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057945862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T09:24:46,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057946071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057946071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057946071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057946072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057946072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T09:24:46,079 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T09:24:46,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/29996e0011a04e15838548fe6f0dece6 2024-11-08T09:24:46,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/7cee5ed119d5462fa7a17fc40cbc136d is 50, key is test_row_0/B:col10/1731057885706/Put/seqid=0 2024-11-08T09:24:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742250_1426 (size=12151) 2024-11-08T09:24:46,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:46,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:46,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T09:24:46,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:46,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:46,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057946381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057946381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057946381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057946381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057946392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T09:24:46,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/7cee5ed119d5462fa7a17fc40cbc136d 2024-11-08T09:24:46,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/18904c67a04a426585dc5d082e9c0664 is 50, key is test_row_0/C:col10/1731057885706/Put/seqid=0 2024-11-08T09:24:46,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742251_1427 (size=12151) 2024-11-08T09:24:46,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057946891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057946901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057946901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057946901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057946912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:46,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:46,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:46,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:46,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/18904c67a04a426585dc5d082e9c0664 2024-11-08T09:24:46,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/29996e0011a04e15838548fe6f0dece6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/29996e0011a04e15838548fe6f0dece6 2024-11-08T09:24:46,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/29996e0011a04e15838548fe6f0dece6, entries=200, sequenceid=140, filesize=14.2 K 2024-11-08T09:24:46,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/7cee5ed119d5462fa7a17fc40cbc136d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7cee5ed119d5462fa7a17fc40cbc136d 2024-11-08T09:24:46,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7cee5ed119d5462fa7a17fc40cbc136d, entries=150, sequenceid=140, filesize=11.9 K 2024-11-08T09:24:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/18904c67a04a426585dc5d082e9c0664 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/18904c67a04a426585dc5d082e9c0664 2024-11-08T09:24:46,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/18904c67a04a426585dc5d082e9c0664, entries=150, sequenceid=140, filesize=11.9 K 2024-11-08T09:24:46,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1246ms, sequenceid=140, compaction requested=true 2024-11-08T09:24:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:46,953 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:46,953 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:46,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:46,954 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:46,954 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:46,954 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/A is initiating minor compaction (all files) 2024-11-08T09:24:46,954 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/B is initiating minor compaction (all files) 2024-11-08T09:24:46,954 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/B in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,954 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/A in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,954 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0cdc590af0974b54becaf6ffb26458e1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/834e74b85ced43a3b99d4efba697e9c4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/29996e0011a04e15838548fe6f0dece6] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=37.9 K 2024-11-08T09:24:46,954 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/82ff505e8ac7473db4506f826cb6fe46, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/757907f2a66d4581803cdfd94da535c0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7cee5ed119d5462fa7a17fc40cbc136d] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=35.5 K 2024-11-08T09:24:46,954 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ff505e8ac7473db4506f826cb6fe46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731057881326 2024-11-08T09:24:46,954 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cdc590af0974b54becaf6ffb26458e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731057881326 2024-11-08T09:24:46,955 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834e74b85ced43a3b99d4efba697e9c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731057883522 2024-11-08T09:24:46,955 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 757907f2a66d4581803cdfd94da535c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731057883522 2024-11-08T09:24:46,955 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29996e0011a04e15838548fe6f0dece6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731057885693 2024-11-08T09:24:46,955 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cee5ed119d5462fa7a17fc40cbc136d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731057885693 2024-11-08T09:24:46,960 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#A#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:46,960 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:46,960 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/e818ffbd0a504e948f3152e92a78f568 is 50, key is test_row_0/A:col10/1731057885706/Put/seqid=0 2024-11-08T09:24:46,961 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/162e90794a3849458699f92689e98e9a is 50, key is test_row_0/B:col10/1731057885706/Put/seqid=0 2024-11-08T09:24:46,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742252_1428 (size=12493) 2024-11-08T09:24:46,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742253_1429 (size=12493) 2024-11-08T09:24:46,974 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/e818ffbd0a504e948f3152e92a78f568 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e818ffbd0a504e948f3152e92a78f568 2024-11-08T09:24:46,978 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/A of 4c6f6f9fc6a2212b0e327dab2405a5f0 into e818ffbd0a504e948f3152e92a78f568(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:46,978 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:46,978 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/A, priority=13, startTime=1731057886953; duration=0sec 2024-11-08T09:24:46,978 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:46,978 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:A 2024-11-08T09:24:46,978 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:46,979 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:46,979 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/C is initiating minor compaction (all files) 2024-11-08T09:24:46,979 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/C in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:46,979 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/2d1664f195f543c5927dd2b8c54f32f7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/caf40f7566974de18ba23f3c2f8012a1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/18904c67a04a426585dc5d082e9c0664] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=35.5 K 2024-11-08T09:24:46,979 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1664f195f543c5927dd2b8c54f32f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731057881326 2024-11-08T09:24:46,980 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf40f7566974de18ba23f3c2f8012a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731057883522 2024-11-08T09:24:46,980 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18904c67a04a426585dc5d082e9c0664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731057885693 2024-11-08T09:24:46,985 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:46,985 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4b7a035c9b4545749e542558fa860db0 is 50, key is test_row_0/C:col10/1731057885706/Put/seqid=0 2024-11-08T09:24:46,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742254_1430 (size=12493) 2024-11-08T09:24:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T09:24:47,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T09:24:47,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:47,072 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:24:47,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/df0ec8da9f8b4626a99f9b5e3ae00698 is 50, key is test_row_0/A:col10/1731057885742/Put/seqid=0 2024-11-08T09:24:47,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742255_1431 (size=12151) 2024-11-08T09:24:47,373 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/162e90794a3849458699f92689e98e9a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/162e90794a3849458699f92689e98e9a 2024-11-08T09:24:47,377 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/B of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 162e90794a3849458699f92689e98e9a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:47,377 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:47,377 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/B, priority=13, startTime=1731057886953; duration=0sec 2024-11-08T09:24:47,377 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:47,378 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:B 2024-11-08T09:24:47,392 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4b7a035c9b4545749e542558fa860db0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4b7a035c9b4545749e542558fa860db0 2024-11-08T09:24:47,395 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/C of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 4b7a035c9b4545749e542558fa860db0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:47,395 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:47,395 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/C, priority=13, startTime=1731057886953; duration=0sec 2024-11-08T09:24:47,395 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:47,395 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:C 2024-11-08T09:24:47,481 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/df0ec8da9f8b4626a99f9b5e3ae00698 2024-11-08T09:24:47,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/1cfc60a2fe37420ba80768f4b5efdb03 is 50, key is test_row_0/B:col10/1731057885742/Put/seqid=0 2024-11-08T09:24:47,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742256_1432 (size=12151) 2024-11-08T09:24:47,892 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/1cfc60a2fe37420ba80768f4b5efdb03 2024-11-08T09:24:47,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/46522581ab174b819dcc2a9f3b8c00da is 50, key is test_row_0/C:col10/1731057885742/Put/seqid=0 2024-11-08T09:24:47,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742257_1433 (size=12151) 2024-11-08T09:24:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:47,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057947925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057947925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057947926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057947928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057947931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:47,936 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T09:24:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057948031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057948041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057948041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057948041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057948242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057948251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057948251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057948251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,300 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/46522581ab174b819dcc2a9f3b8c00da 2024-11-08T09:24:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/df0ec8da9f8b4626a99f9b5e3ae00698 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/df0ec8da9f8b4626a99f9b5e3ae00698 2024-11-08T09:24:48,307 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/df0ec8da9f8b4626a99f9b5e3ae00698, entries=150, sequenceid=165, filesize=11.9 K 2024-11-08T09:24:48,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/1cfc60a2fe37420ba80768f4b5efdb03 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/1cfc60a2fe37420ba80768f4b5efdb03 2024-11-08T09:24:48,311 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/1cfc60a2fe37420ba80768f4b5efdb03, entries=150, sequenceid=165, filesize=11.9 K 2024-11-08T09:24:48,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/46522581ab174b819dcc2a9f3b8c00da as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/46522581ab174b819dcc2a9f3b8c00da 2024-11-08T09:24:48,315 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/46522581ab174b819dcc2a9f3b8c00da, entries=150, sequenceid=165, filesize=11.9 K 2024-11-08T09:24:48,315 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1243ms, sequenceid=165, compaction requested=false 2024-11-08T09:24:48,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:48,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:48,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-08T09:24:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-08T09:24:48,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-08T09:24:48,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4620 sec 2024-11-08T09:24:48,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.4670 sec 2024-11-08T09:24:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:48,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:48,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:24:48,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/ba1f6738ff314d1eb7a5718570721f7c is 50, key is test_row_0/A:col10/1731057888554/Put/seqid=0 2024-11-08T09:24:48,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742258_1434 (size=14541) 2024-11-08T09:24:48,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057948603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057948604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057948604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057948604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057948711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057948711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057948721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057948721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057948921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057948921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057948931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057948941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:48,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/ba1f6738ff314d1eb7a5718570721f7c 2024-11-08T09:24:48,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/14caab88b128407aab6a5b2008e2cd50 is 50, key is test_row_0/B:col10/1731057888554/Put/seqid=0 2024-11-08T09:24:48,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742259_1435 (size=12151) 2024-11-08T09:24:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057949232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057949232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057949241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057949251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/14caab88b128407aab6a5b2008e2cd50 2024-11-08T09:24:49,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1515125404494269862c224260051efd is 50, key is test_row_0/C:col10/1731057888554/Put/seqid=0 2024-11-08T09:24:49,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742260_1436 (size=12151) 2024-11-08T09:24:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057949742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057949742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057949751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057949761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:49,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1515125404494269862c224260051efd 2024-11-08T09:24:49,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/ba1f6738ff314d1eb7a5718570721f7c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ba1f6738ff314d1eb7a5718570721f7c 2024-11-08T09:24:49,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ba1f6738ff314d1eb7a5718570721f7c, entries=200, sequenceid=180, filesize=14.2 K 2024-11-08T09:24:49,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/14caab88b128407aab6a5b2008e2cd50 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/14caab88b128407aab6a5b2008e2cd50 2024-11-08T09:24:49,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/14caab88b128407aab6a5b2008e2cd50, entries=150, sequenceid=180, filesize=11.9 K 2024-11-08T09:24:49,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1515125404494269862c224260051efd as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1515125404494269862c224260051efd 2024-11-08T09:24:49,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1515125404494269862c224260051efd, entries=150, sequenceid=180, filesize=11.9 K 2024-11-08T09:24:49,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1241ms, sequenceid=180, compaction requested=true 2024-11-08T09:24:49,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:49,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:49,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:49,796 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:49,796 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:49,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:49,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:49,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:49,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:49,797 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:49,797 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/A is initiating minor compaction (all files) 2024-11-08T09:24:49,797 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:49,797 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/B is initiating minor compaction (all files) 2024-11-08T09:24:49,797 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/A in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:49,797 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/B in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:49,798 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/162e90794a3849458699f92689e98e9a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/1cfc60a2fe37420ba80768f4b5efdb03, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/14caab88b128407aab6a5b2008e2cd50] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=35.9 K 2024-11-08T09:24:49,798 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e818ffbd0a504e948f3152e92a78f568, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/df0ec8da9f8b4626a99f9b5e3ae00698, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ba1f6738ff314d1eb7a5718570721f7c] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=38.3 K 2024-11-08T09:24:49,798 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 162e90794a3849458699f92689e98e9a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731057885693 2024-11-08T09:24:49,798 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e818ffbd0a504e948f3152e92a78f568, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731057885693 2024-11-08T09:24:49,798 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting df0ec8da9f8b4626a99f9b5e3ae00698, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731057885728 2024-11-08T09:24:49,798 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cfc60a2fe37420ba80768f4b5efdb03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731057885728 2024-11-08T09:24:49,799 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 14caab88b128407aab6a5b2008e2cd50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731057887924 2024-11-08T09:24:49,799 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1f6738ff314d1eb7a5718570721f7c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731057887924 2024-11-08T09:24:49,805 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#A#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:49,806 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/6c9198c0d91a4d29bc2aafb073464b10 is 50, key is test_row_0/A:col10/1731057888554/Put/seqid=0 2024-11-08T09:24:49,808 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:49,809 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/11710b61a9fb4dcdbf21d43b2f00df49 is 50, key is test_row_0/B:col10/1731057888554/Put/seqid=0 2024-11-08T09:24:49,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742262_1438 (size=12595) 2024-11-08T09:24:49,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742261_1437 (size=12595) 2024-11-08T09:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:49,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:49,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:24:49,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/ea151d3ac10d489682a93757bb3aad3a is 50, key is test_row_0/A:col10/1731057888603/Put/seqid=0 2024-11-08T09:24:49,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742263_1439 (size=14541) 2024-11-08T09:24:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057950006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T09:24:50,022 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:50,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-08T09:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T09:24:50,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:50,024 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:50,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:50,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057950121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T09:24:50,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,241 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/11710b61a9fb4dcdbf21d43b2f00df49 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/11710b61a9fb4dcdbf21d43b2f00df49 2024-11-08T09:24:50,241 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/6c9198c0d91a4d29bc2aafb073464b10 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/6c9198c0d91a4d29bc2aafb073464b10 2024-11-08T09:24:50,245 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/B of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 11710b61a9fb4dcdbf21d43b2f00df49(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:50,245 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:50,245 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/B, priority=13, startTime=1731057889796; duration=0sec 2024-11-08T09:24:50,245 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:50,245 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:B 2024-11-08T09:24:50,245 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:50,246 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:50,246 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/C is initiating minor compaction (all files) 2024-11-08T09:24:50,246 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/C in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,246 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4b7a035c9b4545749e542558fa860db0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/46522581ab174b819dcc2a9f3b8c00da, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1515125404494269862c224260051efd] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=35.9 K 2024-11-08T09:24:50,246 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b7a035c9b4545749e542558fa860db0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731057885693 2024-11-08T09:24:50,246 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 46522581ab174b819dcc2a9f3b8c00da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731057885728 2024-11-08T09:24:50,247 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 1515125404494269862c224260051efd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731057887924 2024-11-08T09:24:50,252 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:50,252 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/7edfab94236444c6b705688b3af61697 is 50, key is test_row_0/C:col10/1731057888554/Put/seqid=0 2024-11-08T09:24:50,255 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/A of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 6c9198c0d91a4d29bc2aafb073464b10(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:50,255 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:50,255 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/A, priority=13, startTime=1731057889796; duration=0sec 2024-11-08T09:24:50,255 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:50,255 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:A 2024-11-08T09:24:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742264_1440 (size=12595) 2024-11-08T09:24:50,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057950332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T09:24:50,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/ea151d3ac10d489682a93757bb3aad3a 2024-11-08T09:24:50,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/ad89b518d1ef48c6950442cff805b098 is 50, key is test_row_0/B:col10/1731057888603/Put/seqid=0 2024-11-08T09:24:50,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742265_1441 (size=12151) 2024-11-08T09:24:50,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T09:24:50,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057950651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,661 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/7edfab94236444c6b705688b3af61697 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/7edfab94236444c6b705688b3af61697 2024-11-08T09:24:50,664 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/C of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 7edfab94236444c6b705688b3af61697(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:50,664 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:50,664 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/C, priority=13, startTime=1731057889797; duration=0sec 2024-11-08T09:24:50,664 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:50,664 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:C 2024-11-08T09:24:50,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057950751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,757 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057950761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,767 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057950772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057950772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:50,775 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,775 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:50,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/ad89b518d1ef48c6950442cff805b098 2024-11-08T09:24:50,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/26e697ada27c4d89b08e4ab3daa960ee is 50, key is test_row_0/C:col10/1731057888603/Put/seqid=0 2024-11-08T09:24:50,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742266_1442 (size=12151) 2024-11-08T09:24:50,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:50,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:51,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:51,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T09:24:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057951172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:51,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/26e697ada27c4d89b08e4ab3daa960ee 2024-11-08T09:24:51,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/ea151d3ac10d489682a93757bb3aad3a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ea151d3ac10d489682a93757bb3aad3a 2024-11-08T09:24:51,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ea151d3ac10d489682a93757bb3aad3a, entries=200, sequenceid=204, filesize=14.2 K 2024-11-08T09:24:51,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/ad89b518d1ef48c6950442cff805b098 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/ad89b518d1ef48c6950442cff805b098 2024-11-08T09:24:51,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/ad89b518d1ef48c6950442cff805b098, entries=150, sequenceid=204, filesize=11.9 K 2024-11-08T09:24:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/26e697ada27c4d89b08e4ab3daa960ee as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/26e697ada27c4d89b08e4ab3daa960ee 2024-11-08T09:24:51,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/26e697ada27c4d89b08e4ab3daa960ee, entries=150, sequenceid=204, filesize=11.9 K 2024-11-08T09:24:51,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1245ms, sequenceid=204, compaction requested=false 2024-11-08T09:24:51,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:51,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T09:24:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:51,245 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:24:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/cb6b3ebb04d44b3db7297ee6739b227c is 50, key is test_row_0/A:col10/1731057890005/Put/seqid=0 2024-11-08T09:24:51,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742267_1443 (size=12151) 2024-11-08T09:24:51,655 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/cb6b3ebb04d44b3db7297ee6739b227c 2024-11-08T09:24:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/67bdea1087364e208c6412bcb48570b3 is 50, key is test_row_0/B:col10/1731057890005/Put/seqid=0 2024-11-08T09:24:51,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742268_1444 (size=12151) 2024-11-08T09:24:52,065 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/67bdea1087364e208c6412bcb48570b3 2024-11-08T09:24:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4a56fd5e3aa44874858481225125e3b0 is 50, key is test_row_0/C:col10/1731057890005/Put/seqid=0 2024-11-08T09:24:52,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742269_1445 (size=12151) 2024-11-08T09:24:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T09:24:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:52,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057952353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:52,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057952461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:52,476 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4a56fd5e3aa44874858481225125e3b0 2024-11-08T09:24:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/cb6b3ebb04d44b3db7297ee6739b227c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/cb6b3ebb04d44b3db7297ee6739b227c 2024-11-08T09:24:52,483 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/cb6b3ebb04d44b3db7297ee6739b227c, entries=150, sequenceid=219, filesize=11.9 K 2024-11-08T09:24:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/67bdea1087364e208c6412bcb48570b3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/67bdea1087364e208c6412bcb48570b3 2024-11-08T09:24:52,486 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/67bdea1087364e208c6412bcb48570b3, entries=150, sequenceid=219, filesize=11.9 K 2024-11-08T09:24:52,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4a56fd5e3aa44874858481225125e3b0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4a56fd5e3aa44874858481225125e3b0 2024-11-08T09:24:52,490 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4a56fd5e3aa44874858481225125e3b0, entries=150, sequenceid=219, filesize=11.9 K 2024-11-08T09:24:52,490 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1245ms, sequenceid=219, compaction requested=true 2024-11-08T09:24:52,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:52,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:52,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-08T09:24:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-08T09:24:52,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-08T09:24:52,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-11-08T09:24:52,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.4720 sec 2024-11-08T09:24:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:52,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:52,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:24:52,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/0d14fa0151a946a4b95352ebb17d1732 is 50, key is test_row_0/A:col10/1731057892675/Put/seqid=0 2024-11-08T09:24:52,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742270_1446 (size=14541) 2024-11-08T09:24:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057952719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057952761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,768 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:52,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057952781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,788 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:52,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057952791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,798 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057952801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,808 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057952831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:53,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057953041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:53,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/0d14fa0151a946a4b95352ebb17d1732 2024-11-08T09:24:53,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/dfce1cf91c49489b805e44a6560ee9c6 is 50, key is test_row_0/B:col10/1731057892675/Put/seqid=0 2024-11-08T09:24:53,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742271_1447 (size=12151) 2024-11-08T09:24:53,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057953362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:53,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:53,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/dfce1cf91c49489b805e44a6560ee9c6 2024-11-08T09:24:53,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/cb8e9007c3744cf2a5b0b151f00702be is 50, key is test_row_0/C:col10/1731057892675/Put/seqid=0 2024-11-08T09:24:53,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742272_1448 (size=12151) 2024-11-08T09:24:53,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057953882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:53,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/cb8e9007c3744cf2a5b0b151f00702be 2024-11-08T09:24:53,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/0d14fa0151a946a4b95352ebb17d1732 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0d14fa0151a946a4b95352ebb17d1732 2024-11-08T09:24:53,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0d14fa0151a946a4b95352ebb17d1732, entries=200, sequenceid=244, filesize=14.2 K 2024-11-08T09:24:53,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/dfce1cf91c49489b805e44a6560ee9c6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/dfce1cf91c49489b805e44a6560ee9c6 2024-11-08T09:24:53,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/dfce1cf91c49489b805e44a6560ee9c6, entries=150, sequenceid=244, filesize=11.9 K 2024-11-08T09:24:53,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/cb8e9007c3744cf2a5b0b151f00702be as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/cb8e9007c3744cf2a5b0b151f00702be 2024-11-08T09:24:53,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/cb8e9007c3744cf2a5b0b151f00702be, entries=150, sequenceid=244, filesize=11.9 K 2024-11-08T09:24:53,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1246ms, sequenceid=244, compaction requested=true 2024-11-08T09:24:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:53,923 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:53,923 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:53,924 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:53,924 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:53,924 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/B is initiating minor compaction (all files) 2024-11-08T09:24:53,924 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/A is initiating minor compaction (all files) 2024-11-08T09:24:53,924 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/A in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:53,924 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/B in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:53,925 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/11710b61a9fb4dcdbf21d43b2f00df49, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/ad89b518d1ef48c6950442cff805b098, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/67bdea1087364e208c6412bcb48570b3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/dfce1cf91c49489b805e44a6560ee9c6] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=47.9 K 2024-11-08T09:24:53,925 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/6c9198c0d91a4d29bc2aafb073464b10, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ea151d3ac10d489682a93757bb3aad3a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/cb6b3ebb04d44b3db7297ee6739b227c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0d14fa0151a946a4b95352ebb17d1732] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=52.6 K 2024-11-08T09:24:53,925 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9198c0d91a4d29bc2aafb073464b10, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731057887924 2024-11-08T09:24:53,925 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 11710b61a9fb4dcdbf21d43b2f00df49, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731057887924 2024-11-08T09:24:53,925 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea151d3ac10d489682a93757bb3aad3a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731057888602 2024-11-08T09:24:53,925 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting ad89b518d1ef48c6950442cff805b098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731057888602 2024-11-08T09:24:53,925 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bdea1087364e208c6412bcb48570b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731057889975 2024-11-08T09:24:53,925 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb6b3ebb04d44b3db7297ee6739b227c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731057889975 2024-11-08T09:24:53,925 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting dfce1cf91c49489b805e44a6560ee9c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731057892323 2024-11-08T09:24:53,926 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d14fa0151a946a4b95352ebb17d1732, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731057892316 2024-11-08T09:24:53,933 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#A#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:53,933 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:53,933 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/3739f5a9769e44e1bea549737928723c is 50, key is test_row_0/A:col10/1731057892675/Put/seqid=0 2024-11-08T09:24:53,933 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/231592993f0344278f57ac2957988e26 is 50, key is test_row_0/B:col10/1731057892675/Put/seqid=0 2024-11-08T09:24:53,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742273_1449 (size=12731) 2024-11-08T09:24:53,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742274_1450 (size=12731) 2024-11-08T09:24:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T09:24:54,181 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:54,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-08T09:24:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T09:24:54,183 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:54,184 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:54,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T09:24:54,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T09:24:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:54,336 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:24:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/75b69139926d4f7293cee368376b8dbf is 50, key is test_row_0/A:col10/1731057892718/Put/seqid=0 2024-11-08T09:24:54,343 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/3739f5a9769e44e1bea549737928723c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3739f5a9769e44e1bea549737928723c 2024-11-08T09:24:54,346 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/231592993f0344278f57ac2957988e26 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/231592993f0344278f57ac2957988e26 2024-11-08T09:24:54,347 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/A of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 3739f5a9769e44e1bea549737928723c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:54,347 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:54,347 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/A, priority=12, startTime=1731057893923; duration=0sec 2024-11-08T09:24:54,347 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:54,347 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:A 2024-11-08T09:24:54,347 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:24:54,349 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:24:54,349 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/C is initiating minor compaction (all files) 2024-11-08T09:24:54,349 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/C in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:54,350 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/7edfab94236444c6b705688b3af61697, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/26e697ada27c4d89b08e4ab3daa960ee, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4a56fd5e3aa44874858481225125e3b0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/cb8e9007c3744cf2a5b0b151f00702be] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=47.9 K 2024-11-08T09:24:54,350 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7edfab94236444c6b705688b3af61697, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731057887924 2024-11-08T09:24:54,350 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26e697ada27c4d89b08e4ab3daa960ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731057888602 2024-11-08T09:24:54,350 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a56fd5e3aa44874858481225125e3b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731057889975 2024-11-08T09:24:54,351 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb8e9007c3744cf2a5b0b151f00702be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731057892323 2024-11-08T09:24:54,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742275_1451 (size=12151) 2024-11-08T09:24:54,352 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/75b69139926d4f7293cee368376b8dbf 2024-11-08T09:24:54,352 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/B of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 231592993f0344278f57ac2957988e26(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:54,352 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:54,352 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/B, priority=12, startTime=1731057893923; duration=0sec 2024-11-08T09:24:54,353 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:54,353 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:B 2024-11-08T09:24:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/d87a8f094b654c01884b2ba01ca8ef2b is 50, key is test_row_0/B:col10/1731057892718/Put/seqid=0 2024-11-08T09:24:54,359 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:54,359 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/d4b6963eed794a97a8fb2d53f40bbd01 is 50, key is test_row_0/C:col10/1731057892675/Put/seqid=0 2024-11-08T09:24:54,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742276_1452 (size=12731) 2024-11-08T09:24:54,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742277_1453 (size=12151) 2024-11-08T09:24:54,366 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/d4b6963eed794a97a8fb2d53f40bbd01 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d4b6963eed794a97a8fb2d53f40bbd01 2024-11-08T09:24:54,370 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/C of 4c6f6f9fc6a2212b0e327dab2405a5f0 into d4b6963eed794a97a8fb2d53f40bbd01(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:54,370 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:54,370 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/C, priority=12, startTime=1731057893923; duration=0sec 2024-11-08T09:24:54,370 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:54,370 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:C 2024-11-08T09:24:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T09:24:54,654 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T09:24:54,763 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/d87a8f094b654c01884b2ba01ca8ef2b 2024-11-08T09:24:54,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1f92d2f080c2487b92e93bc4726a4aad is 50, key is test_row_0/C:col10/1731057892718/Put/seqid=0 2024-11-08T09:24:54,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742278_1454 (size=12151) 2024-11-08T09:24:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T09:24:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:54,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:55,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057955046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057955161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:55,181 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1f92d2f080c2487b92e93bc4726a4aad 2024-11-08T09:24:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/75b69139926d4f7293cee368376b8dbf as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/75b69139926d4f7293cee368376b8dbf 2024-11-08T09:24:55,245 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/75b69139926d4f7293cee368376b8dbf, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T09:24:55,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/d87a8f094b654c01884b2ba01ca8ef2b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/d87a8f094b654c01884b2ba01ca8ef2b 2024-11-08T09:24:55,249 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/d87a8f094b654c01884b2ba01ca8ef2b, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T09:24:55,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1f92d2f080c2487b92e93bc4726a4aad as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1f92d2f080c2487b92e93bc4726a4aad 2024-11-08T09:24:55,253 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1f92d2f080c2487b92e93bc4726a4aad, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T09:24:55,253 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 917ms, sequenceid=255, compaction requested=false 2024-11-08T09:24:55,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:55,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-08T09:24:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-08T09:24:55,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-08T09:24:55,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2024-11-08T09:24:55,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.0750 sec 2024-11-08T09:24:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T09:24:55,322 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:55,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:24:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-08T09:24:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T09:24:55,323 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:24:55,324 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:24:55,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:24:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:55,373 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:55,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T09:24:55,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/e60e0685f76c4788b3d6deb807f48616 is 50, key is test_row_0/A:col10/1731057895022/Put/seqid=0 2024-11-08T09:24:55,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742279_1455 (size=14741) 2024-11-08T09:24:55,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/e60e0685f76c4788b3d6deb807f48616 2024-11-08T09:24:55,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/b3f02c8f9827481d9855db44741ce50d is 50, key is test_row_0/B:col10/1731057895022/Put/seqid=0 2024-11-08T09:24:55,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742280_1456 (size=12301) 2024-11-08T09:24:55,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/b3f02c8f9827481d9855db44741ce50d 2024-11-08T09:24:55,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/93b79185336342f480d53273bee87915 is 50, key is test_row_0/C:col10/1731057895022/Put/seqid=0 2024-11-08T09:24:55,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742281_1457 (size=12301) 2024-11-08T09:24:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057955410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T09:24:55,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T09:24:55,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:55,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057955521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:55,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T09:24:55,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:55,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T09:24:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057955741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:55,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T09:24:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:24:55,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/93b79185336342f480d53273bee87915 2024-11-08T09:24:55,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/e60e0685f76c4788b3d6deb807f48616 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e60e0685f76c4788b3d6deb807f48616 2024-11-08T09:24:55,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e60e0685f76c4788b3d6deb807f48616, entries=200, sequenceid=284, filesize=14.4 K 2024-11-08T09:24:55,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/b3f02c8f9827481d9855db44741ce50d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3f02c8f9827481d9855db44741ce50d 2024-11-08T09:24:55,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3f02c8f9827481d9855db44741ce50d, entries=150, sequenceid=284, filesize=12.0 K 2024-11-08T09:24:55,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/93b79185336342f480d53273bee87915 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/93b79185336342f480d53273bee87915 2024-11-08T09:24:55,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/93b79185336342f480d53273bee87915, entries=150, sequenceid=284, filesize=12.0 K 2024-11-08T09:24:55,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 448ms, sequenceid=284, compaction requested=true 2024-11-08T09:24:55,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:55,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:55,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:55,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:55,822 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:55,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:55,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:55,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T09:24:55,822 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:55,823 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:55,823 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/A is initiating minor compaction (all files) 2024-11-08T09:24:55,823 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/A in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,823 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3739f5a9769e44e1bea549737928723c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/75b69139926d4f7293cee368376b8dbf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e60e0685f76c4788b3d6deb807f48616] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=38.7 K 2024-11-08T09:24:55,824 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:55,824 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/B is initiating minor compaction (all files) 2024-11-08T09:24:55,824 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/B in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,824 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/231592993f0344278f57ac2957988e26, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/d87a8f094b654c01884b2ba01ca8ef2b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3f02c8f9827481d9855db44741ce50d] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=36.3 K 2024-11-08T09:24:55,824 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3739f5a9769e44e1bea549737928723c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731057892323 2024-11-08T09:24:55,824 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b69139926d4f7293cee368376b8dbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731057892698 2024-11-08T09:24:55,824 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 231592993f0344278f57ac2957988e26, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731057892323 2024-11-08T09:24:55,825 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting d87a8f094b654c01884b2ba01ca8ef2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731057892698 2024-11-08T09:24:55,825 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60e0685f76c4788b3d6deb807f48616, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057895022 2024-11-08T09:24:55,825 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f02c8f9827481d9855db44741ce50d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057895022 2024-11-08T09:24:55,833 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:55,834 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/bdf45196a187465bbf711137c57d554f is 50, key is test_row_0/B:col10/1731057895022/Put/seqid=0 2024-11-08T09:24:55,834 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:55,834 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/c3dab6bed2194798b21f4c3b21bf289f is 50, key is test_row_0/A:col10/1731057895022/Put/seqid=0 2024-11-08T09:24:55,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742283_1459 (size=12983) 2024-11-08T09:24:55,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742282_1458 (size=12983) 2024-11-08T09:24:55,853 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/c3dab6bed2194798b21f4c3b21bf289f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c3dab6bed2194798b21f4c3b21bf289f 2024-11-08T09:24:55,859 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/bdf45196a187465bbf711137c57d554f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/bdf45196a187465bbf711137c57d554f 2024-11-08T09:24:55,859 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/A of 4c6f6f9fc6a2212b0e327dab2405a5f0 into c3dab6bed2194798b21f4c3b21bf289f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:55,860 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:55,860 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/A, priority=13, startTime=1731057895822; duration=0sec 2024-11-08T09:24:55,860 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:55,860 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:A 2024-11-08T09:24:55,860 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:55,861 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:55,861 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/C is initiating minor compaction (all files) 2024-11-08T09:24:55,861 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/C in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,861 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d4b6963eed794a97a8fb2d53f40bbd01, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1f92d2f080c2487b92e93bc4726a4aad, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/93b79185336342f480d53273bee87915] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=36.3 K 2024-11-08T09:24:55,862 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b6963eed794a97a8fb2d53f40bbd01, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731057892323 2024-11-08T09:24:55,863 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/B of 4c6f6f9fc6a2212b0e327dab2405a5f0 into bdf45196a187465bbf711137c57d554f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:55,863 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:55,863 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f92d2f080c2487b92e93bc4726a4aad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731057892698 2024-11-08T09:24:55,863 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/B, priority=13, startTime=1731057895822; duration=0sec 2024-11-08T09:24:55,863 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:55,863 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:B 2024-11-08T09:24:55,863 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93b79185336342f480d53273bee87915, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057895022 2024-11-08T09:24:55,869 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:55,870 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4d1b8422367c49c69db0486e7c82847b is 50, key is test_row_0/C:col10/1731057895022/Put/seqid=0 2024-11-08T09:24:55,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742284_1460 (size=12983) 2024-11-08T09:24:55,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T09:24:55,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:55,935 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T09:24:55,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/04aad1a3bbad4749818c44e7849d0850 is 50, key is test_row_0/A:col10/1731057895409/Put/seqid=0 2024-11-08T09:24:55,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742285_1461 (size=12301) 2024-11-08T09:24:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T09:24:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:56,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. as already flushing 2024-11-08T09:24:56,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057956221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:56,283 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/4d1b8422367c49c69db0486e7c82847b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4d1b8422367c49c69db0486e7c82847b 2024-11-08T09:24:56,288 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/C of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 4d1b8422367c49c69db0486e7c82847b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:56,288 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:56,288 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/C, priority=13, startTime=1731057895822; duration=0sec 2024-11-08T09:24:56,288 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:56,288 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:C 2024-11-08T09:24:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057956331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:56,353 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/04aad1a3bbad4749818c44e7849d0850 2024-11-08T09:24:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/7ac7e455c5db45bd861698e330896e16 is 50, key is test_row_0/B:col10/1731057895409/Put/seqid=0 2024-11-08T09:24:56,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742286_1462 (size=12301) 2024-11-08T09:24:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T09:24:56,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057956542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:56,762 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/7ac7e455c5db45bd861698e330896e16 2024-11-08T09:24:56,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1eb1c8f8c24c4e9db62312cdbc347943 is 50, key is test_row_0/C:col10/1731057895409/Put/seqid=0 2024-11-08T09:24:56,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742287_1463 (size=12301) 2024-11-08T09:24:56,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32970 deadline: 1731057956791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,798 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32984 deadline: 1731057956812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32936 deadline: 1731057956812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,814 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:56,814 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731057956831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,837 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 4c6f6f9fc6a2212b0e327dab2405a5f0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:24:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:56,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:24:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33008 deadline: 1731057956851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 2024-11-08T09:24:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 , the old value is region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:24:56,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., hostname=cf4d76213909,39151,1731057746690, seqNum=2 because the exception is null or not the one we care about 2024-11-08T09:24:57,170 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1eb1c8f8c24c4e9db62312cdbc347943 2024-11-08T09:24:57,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/04aad1a3bbad4749818c44e7849d0850 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/04aad1a3bbad4749818c44e7849d0850 2024-11-08T09:24:57,176 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/04aad1a3bbad4749818c44e7849d0850, entries=150, sequenceid=296, filesize=12.0 K 2024-11-08T09:24:57,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/7ac7e455c5db45bd861698e330896e16 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7ac7e455c5db45bd861698e330896e16 2024-11-08T09:24:57,180 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7ac7e455c5db45bd861698e330896e16, entries=150, sequenceid=296, filesize=12.0 K 2024-11-08T09:24:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/1eb1c8f8c24c4e9db62312cdbc347943 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1eb1c8f8c24c4e9db62312cdbc347943 2024-11-08T09:24:57,184 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1eb1c8f8c24c4e9db62312cdbc347943, entries=150, sequenceid=296, filesize=12.0 K 2024-11-08T09:24:57,185 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1250ms, sequenceid=296, compaction requested=false 2024-11-08T09:24:57,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:57,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:57,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-08T09:24:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-08T09:24:57,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-08T09:24:57,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-11-08T09:24:57,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8660 sec 2024-11-08T09:24:57,253 INFO [Thread-1824 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1824. 2024-11-08T09:24:57,253 DEBUG [Thread-1824 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:57,253 DEBUG [Thread-1824 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,253 DEBUG [Thread-1824 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:57,253 INFO [Thread-1826 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1826. 2024-11-08T09:24:57,253 DEBUG [Thread-1826 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:57,253 DEBUG [Thread-1826 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,253 DEBUG [Thread-1826 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:57,258 INFO [Thread-1818 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1818. 2024-11-08T09:24:57,258 DEBUG [Thread-1818 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:57,258 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,258 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:57,258 INFO [Thread-1822 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1822. 2024-11-08T09:24:57,258 DEBUG [Thread-1822 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:57,258 DEBUG [Thread-1822 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,259 DEBUG [Thread-1822 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:57,259 INFO [Thread-1820 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1820. 2024-11-08T09:24:57,259 DEBUG [Thread-1820 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:57,259 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,259 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:24:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:24:57,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T09:24:57,363 INFO [Thread-1809 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1809. 2024-11-08T09:24:57,363 DEBUG [Thread-1809 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:24:57,363 DEBUG [Thread-1809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,364 DEBUG [Thread-1809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:24:57,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:24:57,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/c10b745beff84457b698c4cdda98bb43 is 50, key is test_row_0/A:col10/1731057897362/Put/seqid=0 2024-11-08T09:24:57,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742288_1464 (size=12301) 2024-11-08T09:24:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T09:24:57,484 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:24:57,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/c10b745beff84457b698c4cdda98bb43 2024-11-08T09:24:57,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/9c85071fb5564f49a56a0a5a73be50f8 is 50, key is test_row_0/B:col10/1731057897362/Put/seqid=0 2024-11-08T09:24:57,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742289_1465 (size=12301) 2024-11-08T09:24:58,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/9c85071fb5564f49a56a0a5a73be50f8 2024-11-08T09:24:58,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/e2b79eae7f5a436fa12ffe1d35d76e52 is 50, key is test_row_0/C:col10/1731057897362/Put/seqid=0 2024-11-08T09:24:58,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742290_1466 (size=12301) 2024-11-08T09:24:58,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/e2b79eae7f5a436fa12ffe1d35d76e52 2024-11-08T09:24:58,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/c10b745beff84457b698c4cdda98bb43 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c10b745beff84457b698c4cdda98bb43 2024-11-08T09:24:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c10b745beff84457b698c4cdda98bb43, entries=150, sequenceid=324, filesize=12.0 K 2024-11-08T09:24:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/9c85071fb5564f49a56a0a5a73be50f8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/9c85071fb5564f49a56a0a5a73be50f8 2024-11-08T09:24:58,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/9c85071fb5564f49a56a0a5a73be50f8, entries=150, sequenceid=324, filesize=12.0 K 2024-11-08T09:24:58,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/e2b79eae7f5a436fa12ffe1d35d76e52 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/e2b79eae7f5a436fa12ffe1d35d76e52 2024-11-08T09:24:58,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/e2b79eae7f5a436fa12ffe1d35d76e52, entries=150, sequenceid=324, filesize=12.0 K 2024-11-08T09:24:58,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=0 B/0 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1276ms, sequenceid=324, compaction requested=true 2024-11-08T09:24:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:24:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:58,639 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:24:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c6f6f9fc6a2212b0e327dab2405a5f0:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:24:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:58,639 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/A is initiating minor compaction (all files) 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/B is initiating minor compaction (all files) 2024-11-08T09:24:58,640 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/A in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:58,640 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/B in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:58,640 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/bdf45196a187465bbf711137c57d554f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7ac7e455c5db45bd861698e330896e16, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/9c85071fb5564f49a56a0a5a73be50f8] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=36.7 K 2024-11-08T09:24:58,640 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c3dab6bed2194798b21f4c3b21bf289f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/04aad1a3bbad4749818c44e7849d0850, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c10b745beff84457b698c4cdda98bb43] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=36.7 K 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf45196a187465bbf711137c57d554f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057895022 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3dab6bed2194798b21f4c3b21bf289f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057895022 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04aad1a3bbad4749818c44e7849d0850, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731057895383 2024-11-08T09:24:58,640 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac7e455c5db45bd861698e330896e16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731057895383 2024-11-08T09:24:58,641 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10b745beff84457b698c4cdda98bb43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057896203 2024-11-08T09:24:58,641 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c85071fb5564f49a56a0a5a73be50f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057896203 2024-11-08T09:24:58,646 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:58,646 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#B#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:58,646 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/8aea28b88eb64e2a8187b93c66c1c79e is 50, key is test_row_0/A:col10/1731057897362/Put/seqid=0 2024-11-08T09:24:58,646 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/6f04844989fe43ca9cb268bcac56c61b is 50, key is test_row_0/B:col10/1731057897362/Put/seqid=0 2024-11-08T09:24:58,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742292_1468 (size=13085) 2024-11-08T09:24:58,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742291_1467 (size=13085) 2024-11-08T09:24:59,061 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/8aea28b88eb64e2a8187b93c66c1c79e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/8aea28b88eb64e2a8187b93c66c1c79e 2024-11-08T09:24:59,061 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/6f04844989fe43ca9cb268bcac56c61b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/6f04844989fe43ca9cb268bcac56c61b 2024-11-08T09:24:59,066 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/B of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 6f04844989fe43ca9cb268bcac56c61b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:59,066 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/A of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 8aea28b88eb64e2a8187b93c66c1c79e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:59,066 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:59,066 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:59,066 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/B, priority=13, startTime=1731057898639; duration=0sec 2024-11-08T09:24:59,066 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/A, priority=13, startTime=1731057898639; duration=0sec 2024-11-08T09:24:59,067 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:24:59,067 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:B 2024-11-08T09:24:59,067 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:59,067 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:A 2024-11-08T09:24:59,067 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:24:59,068 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:24:59,068 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 4c6f6f9fc6a2212b0e327dab2405a5f0/C is initiating minor compaction (all files) 2024-11-08T09:24:59,068 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c6f6f9fc6a2212b0e327dab2405a5f0/C in TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:24:59,068 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4d1b8422367c49c69db0486e7c82847b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1eb1c8f8c24c4e9db62312cdbc347943, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/e2b79eae7f5a436fa12ffe1d35d76e52] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp, totalSize=36.7 K 2024-11-08T09:24:59,069 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1b8422367c49c69db0486e7c82847b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731057895022 2024-11-08T09:24:59,069 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb1c8f8c24c4e9db62312cdbc347943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731057895383 2024-11-08T09:24:59,069 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b79eae7f5a436fa12ffe1d35d76e52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731057896203 2024-11-08T09:24:59,077 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c6f6f9fc6a2212b0e327dab2405a5f0#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:24:59,078 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/6cf79a3a02a14a1ba608be59d4936b57 is 50, key is test_row_0/C:col10/1731057897362/Put/seqid=0 2024-11-08T09:24:59,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742293_1469 (size=13085) 2024-11-08T09:24:59,494 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/6cf79a3a02a14a1ba608be59d4936b57 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/6cf79a3a02a14a1ba608be59d4936b57 2024-11-08T09:24:59,499 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c6f6f9fc6a2212b0e327dab2405a5f0/C of 4c6f6f9fc6a2212b0e327dab2405a5f0 into 6cf79a3a02a14a1ba608be59d4936b57(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:24:59,499 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: 2024-11-08T09:24:59,499 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., storeName=4c6f6f9fc6a2212b0e327dab2405a5f0/C, priority=13, startTime=1731057898639; duration=0sec 2024-11-08T09:24:59,499 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:24:59,499 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c6f6f9fc6a2212b0e327dab2405a5f0:C 2024-11-08T09:25:06,845 INFO [Thread-1815 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1815. 2024-11-08T09:25:06,845 DEBUG [Thread-1815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:06,845 DEBUG [Thread-1815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,845 DEBUG [Thread-1815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:06,864 INFO [Thread-1811 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1811. 2024-11-08T09:25:06,864 DEBUG [Thread-1811 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:06,864 DEBUG [Thread-1811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,864 DEBUG [Thread-1811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:06,885 INFO [Thread-1807 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1807. 2024-11-08T09:25:06,885 DEBUG [Thread-1807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:06,885 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,886 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,886 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:06,905 INFO [Thread-1813 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1813. 2024-11-08T09:25:06,905 DEBUG [Thread-1813 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:06,906 DEBUG [Thread-1813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,906 DEBUG [Thread-1813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:06,906 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1798 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5392 rows 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1799 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5392 rows 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1799 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5389 rows 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1812 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5434 rows 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1792 2024-11-08T09:25:06,907 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5375 rows 2024-11-08T09:25:06,908 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:25:06,908 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:25:06,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:06,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:06,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T09:25:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:06,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T09:25:06,913 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057906913"}]},"ts":"1731057906913"} 2024-11-08T09:25:06,915 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T09:25:06,915 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T09:25:06,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:25:06,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, UNASSIGN}] 2024-11-08T09:25:06,918 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, UNASSIGN 2024-11-08T09:25:06,919 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=4c6f6f9fc6a2212b0e327dab2405a5f0, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:25:06,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, UNASSIGN because future has completed 2024-11-08T09:25:06,920 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:25:06,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:25:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T09:25:07,074 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:25:07,074 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:25:07,074 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 4c6f6f9fc6a2212b0e327dab2405a5f0, disabling compactions & flushes 2024-11-08T09:25:07,075 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:25:07,075 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:25:07,075 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. after waiting 0 ms 2024-11-08T09:25:07,075 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:25:07,075 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T09:25:07,084 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/2f9e3478965d44f79397c9964a305063 is 50, key is test_row_0/A:col10/1731057906902/Put/seqid=0 2024-11-08T09:25:07,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742294_1470 (size=12301) 2024-11-08T09:25:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T09:25:07,489 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/2f9e3478965d44f79397c9964a305063 2024-11-08T09:25:07,504 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/340e1aa83cc64a4392804e8cdd654783 is 50, key is test_row_0/B:col10/1731057906902/Put/seqid=0 2024-11-08T09:25:07,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742295_1471 (size=12301) 2024-11-08T09:25:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T09:25:07,909 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/340e1aa83cc64a4392804e8cdd654783 2024-11-08T09:25:07,924 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/61857525f4e54d2585cd9a700a9558a8 is 50, key is test_row_0/C:col10/1731057906902/Put/seqid=0 2024-11-08T09:25:07,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742296_1472 (size=12301) 2024-11-08T09:25:08,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T09:25:08,331 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/61857525f4e54d2585cd9a700a9558a8 2024-11-08T09:25:08,342 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/A/2f9e3478965d44f79397c9964a305063 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/2f9e3478965d44f79397c9964a305063 2024-11-08T09:25:08,347 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/2f9e3478965d44f79397c9964a305063, entries=150, sequenceid=334, filesize=12.0 K 2024-11-08T09:25:08,348 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/B/340e1aa83cc64a4392804e8cdd654783 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/340e1aa83cc64a4392804e8cdd654783 2024-11-08T09:25:08,351 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/340e1aa83cc64a4392804e8cdd654783, entries=150, sequenceid=334, filesize=12.0 K 2024-11-08T09:25:08,352 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/.tmp/C/61857525f4e54d2585cd9a700a9558a8 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/61857525f4e54d2585cd9a700a9558a8 2024-11-08T09:25:08,356 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/61857525f4e54d2585cd9a700a9558a8, entries=150, sequenceid=334, filesize=12.0 K 2024-11-08T09:25:08,357 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1282ms, sequenceid=334, compaction requested=false 2024-11-08T09:25:08,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3dc030da12b1480aa2b764f6ee50b161, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/11578fc1246d49349e33c2b1d27a732c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/b1ce16fa0c1e4905af88ea6a16499f7e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/339c95bbc27344c09cb868467af7bfe2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/fe7ec362ba2541949117b1b1f579bf54, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/32326d1094b74e7a9ce2389227965a8d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0cdc590af0974b54becaf6ffb26458e1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/a44e6729b5cd4771974dd7213f09ec6a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/834e74b85ced43a3b99d4efba697e9c4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/29996e0011a04e15838548fe6f0dece6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e818ffbd0a504e948f3152e92a78f568, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/df0ec8da9f8b4626a99f9b5e3ae00698, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ba1f6738ff314d1eb7a5718570721f7c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/6c9198c0d91a4d29bc2aafb073464b10, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ea151d3ac10d489682a93757bb3aad3a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/cb6b3ebb04d44b3db7297ee6739b227c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0d14fa0151a946a4b95352ebb17d1732, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3739f5a9769e44e1bea549737928723c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/75b69139926d4f7293cee368376b8dbf, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e60e0685f76c4788b3d6deb807f48616, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c3dab6bed2194798b21f4c3b21bf289f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/04aad1a3bbad4749818c44e7849d0850, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c10b745beff84457b698c4cdda98bb43] to archive 2024-11-08T09:25:08,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:25:08,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3dc030da12b1480aa2b764f6ee50b161 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3dc030da12b1480aa2b764f6ee50b161 2024-11-08T09:25:08,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/11578fc1246d49349e33c2b1d27a732c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/11578fc1246d49349e33c2b1d27a732c 2024-11-08T09:25:08,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/b1ce16fa0c1e4905af88ea6a16499f7e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/b1ce16fa0c1e4905af88ea6a16499f7e 2024-11-08T09:25:08,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/339c95bbc27344c09cb868467af7bfe2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/339c95bbc27344c09cb868467af7bfe2 2024-11-08T09:25:08,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/fe7ec362ba2541949117b1b1f579bf54 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/fe7ec362ba2541949117b1b1f579bf54 2024-11-08T09:25:08,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/32326d1094b74e7a9ce2389227965a8d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/32326d1094b74e7a9ce2389227965a8d 2024-11-08T09:25:08,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0cdc590af0974b54becaf6ffb26458e1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0cdc590af0974b54becaf6ffb26458e1 2024-11-08T09:25:08,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/a44e6729b5cd4771974dd7213f09ec6a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/a44e6729b5cd4771974dd7213f09ec6a 2024-11-08T09:25:08,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/834e74b85ced43a3b99d4efba697e9c4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/834e74b85ced43a3b99d4efba697e9c4 2024-11-08T09:25:08,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/29996e0011a04e15838548fe6f0dece6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/29996e0011a04e15838548fe6f0dece6 2024-11-08T09:25:08,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e818ffbd0a504e948f3152e92a78f568 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e818ffbd0a504e948f3152e92a78f568 2024-11-08T09:25:08,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/df0ec8da9f8b4626a99f9b5e3ae00698 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/df0ec8da9f8b4626a99f9b5e3ae00698 2024-11-08T09:25:08,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ba1f6738ff314d1eb7a5718570721f7c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ba1f6738ff314d1eb7a5718570721f7c 2024-11-08T09:25:08,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/6c9198c0d91a4d29bc2aafb073464b10 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/6c9198c0d91a4d29bc2aafb073464b10 2024-11-08T09:25:08,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ea151d3ac10d489682a93757bb3aad3a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/ea151d3ac10d489682a93757bb3aad3a 2024-11-08T09:25:08,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/cb6b3ebb04d44b3db7297ee6739b227c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/cb6b3ebb04d44b3db7297ee6739b227c 2024-11-08T09:25:08,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0d14fa0151a946a4b95352ebb17d1732 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/0d14fa0151a946a4b95352ebb17d1732 2024-11-08T09:25:08,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3739f5a9769e44e1bea549737928723c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/3739f5a9769e44e1bea549737928723c 2024-11-08T09:25:08,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/75b69139926d4f7293cee368376b8dbf to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/75b69139926d4f7293cee368376b8dbf 2024-11-08T09:25:08,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e60e0685f76c4788b3d6deb807f48616 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/e60e0685f76c4788b3d6deb807f48616 2024-11-08T09:25:08,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c3dab6bed2194798b21f4c3b21bf289f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c3dab6bed2194798b21f4c3b21bf289f 2024-11-08T09:25:08,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/04aad1a3bbad4749818c44e7849d0850 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/04aad1a3bbad4749818c44e7849d0850 2024-11-08T09:25:08,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c10b745beff84457b698c4cdda98bb43 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/c10b745beff84457b698c4cdda98bb43 2024-11-08T09:25:08,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/29b853c2c49643f0bed3aa5e89708556, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/3d590d0f7fc74633a91a1997ceca0ee5, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/70ac0e8752d049c5bae73a12de2452d4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3844fe184154e3299aac4f875ab3ad0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/02549750bb7740fe8355c73e3eb96d8f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/fa5efcc4807943619b16c6d34539c26f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/82ff505e8ac7473db4506f826cb6fe46, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/a4e2b029a33c4f2f9f437adb74714e76, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/757907f2a66d4581803cdfd94da535c0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/162e90794a3849458699f92689e98e9a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7cee5ed119d5462fa7a17fc40cbc136d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/1cfc60a2fe37420ba80768f4b5efdb03, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/11710b61a9fb4dcdbf21d43b2f00df49, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/14caab88b128407aab6a5b2008e2cd50, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/ad89b518d1ef48c6950442cff805b098, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/67bdea1087364e208c6412bcb48570b3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/231592993f0344278f57ac2957988e26, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/dfce1cf91c49489b805e44a6560ee9c6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/d87a8f094b654c01884b2ba01ca8ef2b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/bdf45196a187465bbf711137c57d554f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3f02c8f9827481d9855db44741ce50d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7ac7e455c5db45bd861698e330896e16, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/9c85071fb5564f49a56a0a5a73be50f8] to archive 2024-11-08T09:25:08,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:25:08,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/29b853c2c49643f0bed3aa5e89708556 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/29b853c2c49643f0bed3aa5e89708556 2024-11-08T09:25:08,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/3d590d0f7fc74633a91a1997ceca0ee5 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/3d590d0f7fc74633a91a1997ceca0ee5 2024-11-08T09:25:08,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/70ac0e8752d049c5bae73a12de2452d4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/70ac0e8752d049c5bae73a12de2452d4 2024-11-08T09:25:08,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3844fe184154e3299aac4f875ab3ad0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3844fe184154e3299aac4f875ab3ad0 2024-11-08T09:25:08,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/02549750bb7740fe8355c73e3eb96d8f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/02549750bb7740fe8355c73e3eb96d8f 2024-11-08T09:25:08,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/fa5efcc4807943619b16c6d34539c26f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/fa5efcc4807943619b16c6d34539c26f 2024-11-08T09:25:08,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/82ff505e8ac7473db4506f826cb6fe46 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/82ff505e8ac7473db4506f826cb6fe46 2024-11-08T09:25:08,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/a4e2b029a33c4f2f9f437adb74714e76 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/a4e2b029a33c4f2f9f437adb74714e76 2024-11-08T09:25:08,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/757907f2a66d4581803cdfd94da535c0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/757907f2a66d4581803cdfd94da535c0 2024-11-08T09:25:08,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/162e90794a3849458699f92689e98e9a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/162e90794a3849458699f92689e98e9a 2024-11-08T09:25:08,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7cee5ed119d5462fa7a17fc40cbc136d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7cee5ed119d5462fa7a17fc40cbc136d 2024-11-08T09:25:08,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/1cfc60a2fe37420ba80768f4b5efdb03 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/1cfc60a2fe37420ba80768f4b5efdb03 2024-11-08T09:25:08,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/11710b61a9fb4dcdbf21d43b2f00df49 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/11710b61a9fb4dcdbf21d43b2f00df49 2024-11-08T09:25:08,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/14caab88b128407aab6a5b2008e2cd50 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/14caab88b128407aab6a5b2008e2cd50 2024-11-08T09:25:08,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/ad89b518d1ef48c6950442cff805b098 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/ad89b518d1ef48c6950442cff805b098 2024-11-08T09:25:08,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/67bdea1087364e208c6412bcb48570b3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/67bdea1087364e208c6412bcb48570b3 2024-11-08T09:25:08,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/231592993f0344278f57ac2957988e26 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/231592993f0344278f57ac2957988e26 2024-11-08T09:25:08,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/dfce1cf91c49489b805e44a6560ee9c6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/dfce1cf91c49489b805e44a6560ee9c6 2024-11-08T09:25:08,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/d87a8f094b654c01884b2ba01ca8ef2b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/d87a8f094b654c01884b2ba01ca8ef2b 2024-11-08T09:25:08,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/bdf45196a187465bbf711137c57d554f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/bdf45196a187465bbf711137c57d554f 2024-11-08T09:25:08,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3f02c8f9827481d9855db44741ce50d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/b3f02c8f9827481d9855db44741ce50d 2024-11-08T09:25:08,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7ac7e455c5db45bd861698e330896e16 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/7ac7e455c5db45bd861698e330896e16 2024-11-08T09:25:08,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/9c85071fb5564f49a56a0a5a73be50f8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/9c85071fb5564f49a56a0a5a73be50f8 2024-11-08T09:25:08,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d90edd9cdde24d348ca3810523d62ef6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1248ea8096064d9195e3f89c457e6d32, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4e501d8691474e7bb446fc9774405de2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/faffac527aac478d89e999f9de03a06d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/f4907ffda6ba4900b47a1ba777d355cd, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/5fab9d3ee0f1429d99fbc9f81f6f6b40, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/2d1664f195f543c5927dd2b8c54f32f7, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/a6c847e547bd490abe803bd52b95dbd0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/caf40f7566974de18ba23f3c2f8012a1, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4b7a035c9b4545749e542558fa860db0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/18904c67a04a426585dc5d082e9c0664, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/46522581ab174b819dcc2a9f3b8c00da, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/7edfab94236444c6b705688b3af61697, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1515125404494269862c224260051efd, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/26e697ada27c4d89b08e4ab3daa960ee, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4a56fd5e3aa44874858481225125e3b0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d4b6963eed794a97a8fb2d53f40bbd01, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/cb8e9007c3744cf2a5b0b151f00702be, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1f92d2f080c2487b92e93bc4726a4aad, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4d1b8422367c49c69db0486e7c82847b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/93b79185336342f480d53273bee87915, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1eb1c8f8c24c4e9db62312cdbc347943, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/e2b79eae7f5a436fa12ffe1d35d76e52] to archive 2024-11-08T09:25:08,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:25:08,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d90edd9cdde24d348ca3810523d62ef6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d90edd9cdde24d348ca3810523d62ef6 2024-11-08T09:25:08,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1248ea8096064d9195e3f89c457e6d32 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1248ea8096064d9195e3f89c457e6d32 2024-11-08T09:25:08,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4e501d8691474e7bb446fc9774405de2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4e501d8691474e7bb446fc9774405de2 2024-11-08T09:25:08,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/faffac527aac478d89e999f9de03a06d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/faffac527aac478d89e999f9de03a06d 2024-11-08T09:25:08,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/f4907ffda6ba4900b47a1ba777d355cd to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/f4907ffda6ba4900b47a1ba777d355cd 2024-11-08T09:25:08,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/5fab9d3ee0f1429d99fbc9f81f6f6b40 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/5fab9d3ee0f1429d99fbc9f81f6f6b40 2024-11-08T09:25:08,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/2d1664f195f543c5927dd2b8c54f32f7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/2d1664f195f543c5927dd2b8c54f32f7 2024-11-08T09:25:08,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/a6c847e547bd490abe803bd52b95dbd0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/a6c847e547bd490abe803bd52b95dbd0 2024-11-08T09:25:08,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/caf40f7566974de18ba23f3c2f8012a1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/caf40f7566974de18ba23f3c2f8012a1 2024-11-08T09:25:08,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4b7a035c9b4545749e542558fa860db0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4b7a035c9b4545749e542558fa860db0 2024-11-08T09:25:08,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/18904c67a04a426585dc5d082e9c0664 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/18904c67a04a426585dc5d082e9c0664 2024-11-08T09:25:08,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/46522581ab174b819dcc2a9f3b8c00da to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/46522581ab174b819dcc2a9f3b8c00da 2024-11-08T09:25:08,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/7edfab94236444c6b705688b3af61697 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/7edfab94236444c6b705688b3af61697 2024-11-08T09:25:08,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1515125404494269862c224260051efd to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1515125404494269862c224260051efd 2024-11-08T09:25:08,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/26e697ada27c4d89b08e4ab3daa960ee to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/26e697ada27c4d89b08e4ab3daa960ee 2024-11-08T09:25:08,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4a56fd5e3aa44874858481225125e3b0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4a56fd5e3aa44874858481225125e3b0 2024-11-08T09:25:08,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d4b6963eed794a97a8fb2d53f40bbd01 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/d4b6963eed794a97a8fb2d53f40bbd01 2024-11-08T09:25:08,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/cb8e9007c3744cf2a5b0b151f00702be to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/cb8e9007c3744cf2a5b0b151f00702be 2024-11-08T09:25:08,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1f92d2f080c2487b92e93bc4726a4aad to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1f92d2f080c2487b92e93bc4726a4aad 2024-11-08T09:25:08,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4d1b8422367c49c69db0486e7c82847b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/4d1b8422367c49c69db0486e7c82847b 2024-11-08T09:25:08,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/93b79185336342f480d53273bee87915 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/93b79185336342f480d53273bee87915 2024-11-08T09:25:08,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1eb1c8f8c24c4e9db62312cdbc347943 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/1eb1c8f8c24c4e9db62312cdbc347943 2024-11-08T09:25:08,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/e2b79eae7f5a436fa12ffe1d35d76e52 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/e2b79eae7f5a436fa12ffe1d35d76e52 2024-11-08T09:25:08,444 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=1 2024-11-08T09:25:08,444 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. 2024-11-08T09:25:08,445 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 4c6f6f9fc6a2212b0e327dab2405a5f0: Waiting for close lock at 1731057907074Running coprocessor pre-close hooks at 1731057907074Disabling compacts and flushes for region at 1731057907074Disabling writes for close at 1731057907075 (+1 ms)Obtaining lock to block concurrent updates at 1731057907075Preparing flush snapshotting stores in 4c6f6f9fc6a2212b0e327dab2405a5f0 at 1731057907075Finished memstore snapshotting TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731057907076 (+1 ms)Flushing stores of TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0. at 1731057907077 (+1 ms)Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/A: creating writer at 1731057907078 (+1 ms)Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/A: appending metadata at 1731057907084 (+6 ms)Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/A: closing flushed file at 1731057907084Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/B: creating writer at 1731057907499 (+415 ms)Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/B: appending metadata at 1731057907504 (+5 ms)Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/B: closing flushed file at 1731057907504Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/C: creating writer at 1731057907918 (+414 ms)Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/C: appending metadata at 1731057907923 (+5 ms)Flushing 4c6f6f9fc6a2212b0e327dab2405a5f0/C: closing flushed file at 1731057907923Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57a913e2: reopening flushed file at 1731057908340 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73c7fe23: reopening flushed file at 1731057908347 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48fe1e14: reopening flushed file at 1731057908351 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4c6f6f9fc6a2212b0e327dab2405a5f0 in 1282ms, sequenceid=334, compaction requested=false at 1731057908357 (+6 ms)Writing region close event to WAL at 1731057908442 (+85 ms)Running coprocessor post-close hooks at 1731057908444 (+2 ms)Closed at 1731057908444 2024-11-08T09:25:08,446 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:25:08,447 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=4c6f6f9fc6a2212b0e327dab2405a5f0, regionState=CLOSED 2024-11-08T09:25:08,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:25:08,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-08T09:25:08,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c6f6f9fc6a2212b0e327dab2405a5f0, server=cf4d76213909,39151,1731057746690 in 1.5290 sec 2024-11-08T09:25:08,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-08T09:25:08,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c6f6f9fc6a2212b0e327dab2405a5f0, UNASSIGN in 1.5330 sec 2024-11-08T09:25:08,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-08T09:25:08,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-08T09:25:08,454 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057908454"}]},"ts":"1731057908454"} 2024-11-08T09:25:08,455 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T09:25:08,455 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T09:25:08,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5470 sec 2024-11-08T09:25:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T09:25:09,063 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:09,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T09:25:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:09,070 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:09,072 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T09:25:09,076 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:25:09,079 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/recovered.edits] 2024-11-08T09:25:09,083 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/2f9e3478965d44f79397c9964a305063 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/2f9e3478965d44f79397c9964a305063 2024-11-08T09:25:09,085 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/8aea28b88eb64e2a8187b93c66c1c79e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/A/8aea28b88eb64e2a8187b93c66c1c79e 2024-11-08T09:25:09,087 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/340e1aa83cc64a4392804e8cdd654783 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/340e1aa83cc64a4392804e8cdd654783 2024-11-08T09:25:09,088 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/6f04844989fe43ca9cb268bcac56c61b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/B/6f04844989fe43ca9cb268bcac56c61b 2024-11-08T09:25:09,091 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/61857525f4e54d2585cd9a700a9558a8 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/61857525f4e54d2585cd9a700a9558a8 2024-11-08T09:25:09,092 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/6cf79a3a02a14a1ba608be59d4936b57 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/C/6cf79a3a02a14a1ba608be59d4936b57 2024-11-08T09:25:09,094 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/recovered.edits/337.seqid to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0/recovered.edits/337.seqid 2024-11-08T09:25:09,095 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/4c6f6f9fc6a2212b0e327dab2405a5f0 2024-11-08T09:25:09,095 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T09:25:09,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:09,099 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T09:25:09,100 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T09:25:09,101 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:09,101 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T09:25:09,102 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731057909101"}]},"ts":"9223372036854775807"} 2024-11-08T09:25:09,103 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T09:25:09,103 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4c6f6f9fc6a2212b0e327dab2405a5f0, NAME => 'TestAcidGuarantees,,1731057875050.4c6f6f9fc6a2212b0e327dab2405a5f0.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T09:25:09,103 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T09:25:09,104 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731057909103"}]},"ts":"9223372036854775807"} 2024-11-08T09:25:09,105 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T09:25:09,106 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:09,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-08T09:25:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T09:25:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T09:25:09,182 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:09,193 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 238), OpenFileDescriptor=443 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 305), ProcessCount=11 (was 11), AvailableMemoryMB=5437 (was 5457) 2024-11-08T09:25:09,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=5437 2024-11-08T09:25:09,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:25:09,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:25:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:09,205 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T09:25:09,206 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:09,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-08T09:25:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-08T09:25:09,207 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T09:25:09,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742297_1473 (size=959) 2024-11-08T09:25:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-08T09:25:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-08T09:25:09,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1 2024-11-08T09:25:09,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742298_1474 (size=53) 2024-11-08T09:25:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-08T09:25:10,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:25:10,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8144a32b044f8a5dee9d583d8360a12c, disabling compactions & flushes 2024-11-08T09:25:10,029 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:10,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:10,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. after waiting 0 ms 2024-11-08T09:25:10,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:10,029 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:10,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8144a32b044f8a5dee9d583d8360a12c: Waiting for close lock at 1731057910029Disabling compacts and flushes for region at 1731057910029Disabling writes for close at 1731057910029Writing region close event to WAL at 1731057910029Closed at 1731057910029 2024-11-08T09:25:10,033 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T09:25:10,034 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731057910033"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731057910033"}]},"ts":"1731057910033"} 2024-11-08T09:25:10,036 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T09:25:10,037 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T09:25:10,037 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057910037"}]},"ts":"1731057910037"} 2024-11-08T09:25:10,038 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T09:25:10,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, ASSIGN}] 2024-11-08T09:25:10,039 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, ASSIGN 2024-11-08T09:25:10,040 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, ASSIGN; state=OFFLINE, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=false 2024-11-08T09:25:10,191 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:25:10,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, ASSIGN because future has completed 2024-11-08T09:25:10,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:25:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-08T09:25:10,351 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:10,352 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:25:10,352 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,352 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:25:10,352 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,352 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,353 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,355 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8144a32b044f8a5dee9d583d8360a12c columnFamilyName A 2024-11-08T09:25:10,355 DEBUG [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:10,355 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(327): Store=8144a32b044f8a5dee9d583d8360a12c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:25:10,355 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,356 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8144a32b044f8a5dee9d583d8360a12c columnFamilyName B 2024-11-08T09:25:10,356 DEBUG [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:10,357 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(327): Store=8144a32b044f8a5dee9d583d8360a12c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:25:10,357 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,359 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8144a32b044f8a5dee9d583d8360a12c columnFamilyName C 2024-11-08T09:25:10,359 DEBUG [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:10,359 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(327): Store=8144a32b044f8a5dee9d583d8360a12c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:25:10,359 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,360 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,361 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,362 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,362 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,363 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:25:10,365 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,367 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T09:25:10,368 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened 8144a32b044f8a5dee9d583d8360a12c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59121287, jitterRate=-0.11902417242527008}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:25:10,368 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:10,369 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for 8144a32b044f8a5dee9d583d8360a12c: Running coprocessor pre-open hook at 1731057910352Writing region info on filesystem at 1731057910352Initializing all the Stores at 1731057910353 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057910353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057910353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057910353Cleaning up temporary data from old regions at 1731057910362 (+9 ms)Running coprocessor post-open hooks at 1731057910368 (+6 ms)Region opened successfully at 1731057910369 (+1 ms) 2024-11-08T09:25:10,370 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., pid=134, masterSystemTime=1731057910349 2024-11-08T09:25:10,372 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:10,372 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:10,373 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=OPEN, openSeqNum=2, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:25:10,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:25:10,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-08T09:25:10,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 in 179 msec 2024-11-08T09:25:10,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-08T09:25:10,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, ASSIGN in 340 msec 2024-11-08T09:25:10,381 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T09:25:10,381 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057910381"}]},"ts":"1731057910381"} 2024-11-08T09:25:10,382 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T09:25:10,383 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T09:25:10,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-08T09:25:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-08T09:25:11,352 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:11,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4826bbfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:11,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:11,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:11,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dc98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:11,360 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26930eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:11,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:11,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:11,365 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T09:25:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is cf4d76213909,35807,1731057745600 2024-11-08T09:25:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@437fea34 2024-11-08T09:25:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T09:25:11,367 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T09:25:11,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T09:25:11,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T09:25:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:11,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742299_1475 (size=995) 2024-11-08T09:25:11,785 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T09:25:11,785 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T09:25:11,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:25:11,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, REOPEN/MOVE}] 2024-11-08T09:25:11,796 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, REOPEN/MOVE 2024-11-08T09:25:11,797 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:25:11,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, REOPEN/MOVE because future has completed 2024-11-08T09:25:11,798 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:25:11,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:25:11,951 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:11,951 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:25:11,951 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 8144a32b044f8a5dee9d583d8360a12c, disabling compactions & flushes 2024-11-08T09:25:11,951 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:11,951 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:11,951 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. after waiting 0 ms 2024-11-08T09:25:11,952 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,017 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T09:25:12,018 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,018 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 8144a32b044f8a5dee9d583d8360a12c: Waiting for close lock at 1731057911951Running coprocessor pre-close hooks at 1731057911951Disabling compacts and flushes for region at 1731057911951Disabling writes for close at 1731057911952 (+1 ms)Writing region close event to WAL at 1731057911953 (+1 ms)Running coprocessor post-close hooks at 1731057912018 (+65 ms)Closed at 1731057912018 2024-11-08T09:25:12,019 WARN [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: 8144a32b044f8a5dee9d583d8360a12c to self. 2024-11-08T09:25:12,023 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,024 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=CLOSED 2024-11-08T09:25:12,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:25:12,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-08T09:25:12,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 in 231 msec 2024-11-08T09:25:12,033 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, REOPEN/MOVE; state=CLOSED, location=cf4d76213909,39151,1731057746690; forceNewPlan=false, retain=true 2024-11-08T09:25:12,184 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=OPENING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, REOPEN/MOVE because future has completed 2024-11-08T09:25:12,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:25:12,350 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,350 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T09:25:12,351 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,351 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T09:25:12,351 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,351 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,353 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,354 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8144a32b044f8a5dee9d583d8360a12c columnFamilyName A 2024-11-08T09:25:12,356 DEBUG [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:12,356 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(327): Store=8144a32b044f8a5dee9d583d8360a12c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:25:12,357 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,358 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8144a32b044f8a5dee9d583d8360a12c columnFamilyName B 2024-11-08T09:25:12,358 DEBUG [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:12,358 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(327): Store=8144a32b044f8a5dee9d583d8360a12c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:25:12,358 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,359 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8144a32b044f8a5dee9d583d8360a12c columnFamilyName C 2024-11-08T09:25:12,360 DEBUG [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:12,360 INFO [StoreOpener-8144a32b044f8a5dee9d583d8360a12c-1 {}] regionserver.HStore(327): Store=8144a32b044f8a5dee9d583d8360a12c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T09:25:12,360 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,361 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,363 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,364 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,364 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,364 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T09:25:12,365 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,366 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened 8144a32b044f8a5dee9d583d8360a12c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63240795, jitterRate=-0.05763871967792511}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T09:25:12,366 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,366 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for 8144a32b044f8a5dee9d583d8360a12c: Running coprocessor pre-open hook at 1731057912352Writing region info on filesystem at 1731057912352Initializing all the Stores at 1731057912353 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057912353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057912353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731057912353Cleaning up temporary data from old regions at 1731057912364 (+11 ms)Running coprocessor post-open hooks at 1731057912366 (+2 ms)Region opened successfully at 1731057912366 2024-11-08T09:25:12,367 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., pid=139, masterSystemTime=1731057912343 2024-11-08T09:25:12,369 DEBUG [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,369 INFO [RS_OPEN_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,369 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=OPEN, openSeqNum=5, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:25:12,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-08T09:25:12,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 in 182 msec 2024-11-08T09:25:12,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-08T09:25:12,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, REOPEN/MOVE in 577 msec 2024-11-08T09:25:12,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-08T09:25:12,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 583 msec 2024-11-08T09:25:12,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0080 sec 2024-11-08T09:25:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T09:25:12,380 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:12,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604e524e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,381 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b97e9c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,383 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488ea8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f36d46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,385 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@236ffe42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,386 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2982a584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5e61df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,388 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a4ffb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,390 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5906518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885acdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,393 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45800b78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,395 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c6d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc99027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,398 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1416e901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,400 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab9c85f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3420b4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,403 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9dd4b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47700470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1b2f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a45a24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8cb563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1102b71d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea4815a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757dc6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ff4806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,413 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8d676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,414 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a49718f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8677bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cf4d76213909,35807,-1 for getting cluster id 2024-11-08T09:25:12,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T09:25:12,416 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59c75356-9176-4dc6-a8b1-6b5efa72c120' 2024-11-08T09:25:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T09:25:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59c75356-9176-4dc6-a8b1-6b5efa72c120" 2024-11-08T09:25:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0ee6b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cf4d76213909,35807,-1] 2024-11-08T09:25:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T09:25:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:12,417 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T09:25:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3386c9c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T09:25:12,418 DEBUG [Thread-2162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,418 DEBUG [Thread-2171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,419 DEBUG [Thread-2156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,419 DEBUG [Thread-2167 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,419 DEBUG [Thread-2169 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,419 DEBUG [Thread-2154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,419 DEBUG [Thread-2160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,420 DEBUG [Thread-2169 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,420 DEBUG [Thread-2169 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,424 DEBUG [Thread-2173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,424 DEBUG [Thread-2158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,425 DEBUG [Thread-2165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T09:25:12,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cf4d76213909,39151,1731057746690, seqNum=-1] 2024-11-08T09:25:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T09:25:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T09:25:12,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5] 2024-11-08T09:25:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-08T09:25:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-08T09:25:12,430 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:25:12,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:25:12,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:12,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:12,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:12,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108deb9883977774684a24b79b0a2ea293f_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057912430/Put/seqid=0 2024-11-08T09:25:12,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742300_1476 (size=12154) 2024-11-08T09:25:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057972519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057972520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057972520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057972520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057972521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-08T09:25:12,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:12,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:12,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057972631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057972632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057972632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057972632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057972632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:12,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:12,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-08T09:25:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057972841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057972842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057972842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057972842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057972842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:12,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:12,860 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108deb9883977774684a24b79b0a2ea293f_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108deb9883977774684a24b79b0a2ea293f_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:12,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f308e6bdae8e4a4799594fa8ffb2d827, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:12,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f308e6bdae8e4a4799594fa8ffb2d827 is 175, key is test_row_0/A:col10/1731057912430/Put/seqid=0 2024-11-08T09:25:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742301_1477 (size=30955) 2024-11-08T09:25:12,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:12,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-08T09:25:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057973151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057973152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057973152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057973152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057973152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:13,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:13,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f308e6bdae8e4a4799594fa8ffb2d827 2024-11-08T09:25:13,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/11d02dd39c8e4fa4a8b8a87317d30111 is 50, key is test_row_0/B:col10/1731057912430/Put/seqid=0 2024-11-08T09:25:13,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742302_1478 (size=12001) 2024-11-08T09:25:13,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:13,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-08T09:25:13,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057973661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057973662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057973662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057973662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057973662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:13,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/11d02dd39c8e4fa4a8b8a87317d30111 2024-11-08T09:25:13,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/457e5ae6a9a3499c9efba3e80b90b8af is 50, key is test_row_0/C:col10/1731057912430/Put/seqid=0 2024-11-08T09:25:13,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742303_1479 (size=12001) 2024-11-08T09:25:13,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:13,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:13,961 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T09:25:14,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:14,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:14,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/457e5ae6a9a3499c9efba3e80b90b8af 2024-11-08T09:25:14,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f308e6bdae8e4a4799594fa8ffb2d827 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f308e6bdae8e4a4799594fa8ffb2d827 2024-11-08T09:25:14,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f308e6bdae8e4a4799594fa8ffb2d827, entries=150, sequenceid=16, filesize=30.2 K 2024-11-08T09:25:14,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/11d02dd39c8e4fa4a8b8a87317d30111 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/11d02dd39c8e4fa4a8b8a87317d30111 2024-11-08T09:25:14,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/11d02dd39c8e4fa4a8b8a87317d30111, entries=150, sequenceid=16, filesize=11.7 K 2024-11-08T09:25:14,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/457e5ae6a9a3499c9efba3e80b90b8af as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/457e5ae6a9a3499c9efba3e80b90b8af 2024-11-08T09:25:14,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/457e5ae6a9a3499c9efba3e80b90b8af, entries=150, sequenceid=16, filesize=11.7 K 2024-11-08T09:25:14,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8144a32b044f8a5dee9d583d8360a12c in 1703ms, sequenceid=16, compaction requested=false 2024-11-08T09:25:14,134 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T09:25:14,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:14,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-08T09:25:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:14,266 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:25:14,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082b156d4947f7467c97e80e4acd1befc5_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057912520/Put/seqid=0 2024-11-08T09:25:14,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742304_1480 (size=12154) 2024-11-08T09:25:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-08T09:25:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:14,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:14,679 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082b156d4947f7467c97e80e4acd1befc5_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082b156d4947f7467c97e80e4acd1befc5_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:14,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/e540f93193b84b20837f37edadc7a361, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:14,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/e540f93193b84b20837f37edadc7a361 is 175, key is test_row_0/A:col10/1731057912520/Put/seqid=0 2024-11-08T09:25:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057974678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057974681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057974682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057974682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057974682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,683 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 8144a32b044f8a5dee9d583d8360a12c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:25:14,683 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 8144a32b044f8a5dee9d583d8360a12c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:14,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742305_1481 (size=30955) 2024-11-08T09:25:14,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057974792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057974792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:14,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057974792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:14,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057975001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057975001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057975001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,085 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/e540f93193b84b20837f37edadc7a361 2024-11-08T09:25:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/475aec0dd0c44c3186cce5d25313babc is 50, key is test_row_0/B:col10/1731057912520/Put/seqid=0 2024-11-08T09:25:15,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742306_1482 (size=12001) 2024-11-08T09:25:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057975312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057975312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057975312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,495 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/475aec0dd0c44c3186cce5d25313babc 2024-11-08T09:25:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0d31ee44264d4e9c9a5b2b84b45f693e is 50, key is test_row_0/C:col10/1731057912520/Put/seqid=0 2024-11-08T09:25:15,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742307_1483 (size=12001) 2024-11-08T09:25:15,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057975821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057975822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057975822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:15,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:15,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:15,904 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0d31ee44264d4e9c9a5b2b84b45f693e 2024-11-08T09:25:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/e540f93193b84b20837f37edadc7a361 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/e540f93193b84b20837f37edadc7a361 2024-11-08T09:25:15,911 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/e540f93193b84b20837f37edadc7a361, entries=150, sequenceid=41, filesize=30.2 K 2024-11-08T09:25:15,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/475aec0dd0c44c3186cce5d25313babc as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/475aec0dd0c44c3186cce5d25313babc 2024-11-08T09:25:15,914 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/475aec0dd0c44c3186cce5d25313babc, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T09:25:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0d31ee44264d4e9c9a5b2b84b45f693e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0d31ee44264d4e9c9a5b2b84b45f693e 2024-11-08T09:25:15,918 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0d31ee44264d4e9c9a5b2b84b45f693e, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T09:25:15,919 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8144a32b044f8a5dee9d583d8360a12c in 1653ms, sequenceid=41, compaction requested=false 2024-11-08T09:25:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:15,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-08T09:25:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-08T09:25:15,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-08T09:25:15,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4860 sec 2024-11-08T09:25:15,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 3.4940 sec 2024-11-08T09:25:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-08T09:25:16,601 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:16,602 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-08T09:25:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T09:25:16,603 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:16,604 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:16,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:16,693 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:25:16,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:25:16,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f11d3f0112ac484e9bc977310bd0b4e3_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057914676/Put/seqid=0 2024-11-08T09:25:16,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742308_1484 (size=14594) 2024-11-08T09:25:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T09:25:16,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:16,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057976766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:16,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057976766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:16,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057976831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:16,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057976831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:16,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057976831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:16,833 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_2' in 8144a32b044f8a5dee9d583d8360a12c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:25:16,833 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_0' in 8144a32b044f8a5dee9d583d8360a12c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:16,834 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to cf4d76213909,39151,1731057746690 for 'test_row_1' in 8144a32b044f8a5dee9d583d8360a12c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:16,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057976872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057976872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:16,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:16,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:16,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:16,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:16,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:16,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T09:25:17,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057977082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:17,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057977082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:17,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:17,116 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f11d3f0112ac484e9bc977310bd0b4e3_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f11d3f0112ac484e9bc977310bd0b4e3_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:17,116 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/69a789fbc2384271868b705c5d0f9650, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:17,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/69a789fbc2384271868b705c5d0f9650 is 175, key is test_row_0/A:col10/1731057914676/Put/seqid=0 2024-11-08T09:25:17,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742309_1485 (size=39549) 2024-11-08T09:25:17,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:17,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:17,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T09:25:17,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057977391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:17,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057977391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:17,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/69a789fbc2384271868b705c5d0f9650 2024-11-08T09:25:17,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/9671b1893f8c4f81af18dc970c75a15a is 50, key is test_row_0/B:col10/1731057914676/Put/seqid=0 2024-11-08T09:25:17,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742310_1486 (size=12001) 2024-11-08T09:25:17,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/9671b1893f8c4f81af18dc970c75a15a 2024-11-08T09:25:17,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/191d8d21e26443b09b89fb95ed80f26f is 50, key is test_row_0/C:col10/1731057914676/Put/seqid=0 2024-11-08T09:25:17,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742311_1487 (size=12001) 2024-11-08T09:25:17,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T09:25:17,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057977901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:17,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057977901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:17,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/191d8d21e26443b09b89fb95ed80f26f 2024-11-08T09:25:17,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/69a789fbc2384271868b705c5d0f9650 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/69a789fbc2384271868b705c5d0f9650 2024-11-08T09:25:17,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/69a789fbc2384271868b705c5d0f9650, entries=200, sequenceid=53, filesize=38.6 K 2024-11-08T09:25:17,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/9671b1893f8c4f81af18dc970c75a15a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/9671b1893f8c4f81af18dc970c75a15a 2024-11-08T09:25:17,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/9671b1893f8c4f81af18dc970c75a15a, entries=150, sequenceid=53, filesize=11.7 K 2024-11-08T09:25:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/191d8d21e26443b09b89fb95ed80f26f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/191d8d21e26443b09b89fb95ed80f26f 2024-11-08T09:25:17,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/191d8d21e26443b09b89fb95ed80f26f, entries=150, sequenceid=53, filesize=11.7 K 2024-11-08T09:25:17,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8144a32b044f8a5dee9d583d8360a12c in 1268ms, sequenceid=53, compaction requested=true 2024-11-08T09:25:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:25:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:25:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:25:17,961 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:17,961 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:17,962 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:17,962 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/A is initiating minor compaction (all files) 2024-11-08T09:25:17,962 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:17,962 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/B is initiating minor compaction (all files) 2024-11-08T09:25:17,962 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/A in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,962 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/B in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,962 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f308e6bdae8e4a4799594fa8ffb2d827, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/e540f93193b84b20837f37edadc7a361, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/69a789fbc2384271868b705c5d0f9650] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=99.1 K 2024-11-08T09:25:17,962 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/11d02dd39c8e4fa4a8b8a87317d30111, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/475aec0dd0c44c3186cce5d25313babc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/9671b1893f8c4f81af18dc970c75a15a] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=35.2 K 2024-11-08T09:25:17,962 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,962 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f308e6bdae8e4a4799594fa8ffb2d827, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/e540f93193b84b20837f37edadc7a361, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/69a789fbc2384271868b705c5d0f9650] 2024-11-08T09:25:17,962 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d02dd39c8e4fa4a8b8a87317d30111, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731057912426 2024-11-08T09:25:17,962 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f308e6bdae8e4a4799594fa8ffb2d827, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731057912426 2024-11-08T09:25:17,963 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 475aec0dd0c44c3186cce5d25313babc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731057912518 2024-11-08T09:25:17,963 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting e540f93193b84b20837f37edadc7a361, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731057912518 2024-11-08T09:25:17,963 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 9671b1893f8c4f81af18dc970c75a15a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057914676 2024-11-08T09:25:17,963 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a789fbc2384271868b705c5d0f9650, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057914676 2024-11-08T09:25:17,968 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:17,969 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/eebd6941daf04c208af3b7121eb2cef3 is 50, key is test_row_0/B:col10/1731057914676/Put/seqid=0 2024-11-08T09:25:17,969 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:17,971 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411081290ee5ae4104c61be241bb4a7ca535f_8144a32b044f8a5dee9d583d8360a12c store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:17,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742312_1488 (size=12104) 2024-11-08T09:25:17,974 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411081290ee5ae4104c61be241bb4a7ca535f_8144a32b044f8a5dee9d583d8360a12c, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:17,974 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081290ee5ae4104c61be241bb4a7ca535f_8144a32b044f8a5dee9d583d8360a12c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:17,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-08T09:25:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,977 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:25:17,984 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/eebd6941daf04c208af3b7121eb2cef3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/eebd6941daf04c208af3b7121eb2cef3 2024-11-08T09:25:17,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742313_1489 (size=4469) 2024-11-08T09:25:17,990 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/B of 8144a32b044f8a5dee9d583d8360a12c into eebd6941daf04c208af3b7121eb2cef3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:17,990 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:17,990 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/B, priority=13, startTime=1731057917961; duration=0sec 2024-11-08T09:25:17,990 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:17,990 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:B 2024-11-08T09:25:17,990 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:17,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108628b3086071a4e95833b3a787fcc07f4_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057916744/Put/seqid=0 2024-11-08T09:25:17,995 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:17,995 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/C is initiating minor compaction (all files) 2024-11-08T09:25:17,995 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/C in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:17,996 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/457e5ae6a9a3499c9efba3e80b90b8af, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0d31ee44264d4e9c9a5b2b84b45f693e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/191d8d21e26443b09b89fb95ed80f26f] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=35.2 K 2024-11-08T09:25:17,996 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 457e5ae6a9a3499c9efba3e80b90b8af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731057912426 2024-11-08T09:25:17,996 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d31ee44264d4e9c9a5b2b84b45f693e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731057912518 2024-11-08T09:25:17,997 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 191d8d21e26443b09b89fb95ed80f26f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057914676 2024-11-08T09:25:17,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742314_1490 (size=12154) 2024-11-08T09:25:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,002 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108628b3086071a4e95833b3a787fcc07f4_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108628b3086071a4e95833b3a787fcc07f4_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:18,003 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:18,004 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/00ee38fd5a4d429db03b69a3af396267 is 50, key is test_row_0/C:col10/1731057914676/Put/seqid=0 2024-11-08T09:25:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/9ccbe47603834a1886139b65ec823cd2, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:18,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/9ccbe47603834a1886139b65ec823cd2 is 175, key is test_row_0/A:col10/1731057916744/Put/seqid=0 2024-11-08T09:25:18,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742315_1491 (size=12104) 2024-11-08T09:25:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742316_1492 (size=30955) 2024-11-08T09:25:18,390 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#A#compaction#408 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:18,390 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/9ff818e3e5cf41da8c098de9bdf1ee73 is 175, key is test_row_0/A:col10/1731057914676/Put/seqid=0 2024-11-08T09:25:18,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742317_1493 (size=31058) 2024-11-08T09:25:18,410 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/9ccbe47603834a1886139b65ec823cd2 2024-11-08T09:25:18,412 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/00ee38fd5a4d429db03b69a3af396267 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/00ee38fd5a4d429db03b69a3af396267 2024-11-08T09:25:18,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/67e2352ced37455db36f5042db122d42 is 50, key is test_row_0/B:col10/1731057916744/Put/seqid=0 2024-11-08T09:25:18,420 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/C of 8144a32b044f8a5dee9d583d8360a12c into 00ee38fd5a4d429db03b69a3af396267(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:18,420 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:18,420 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/C, priority=13, startTime=1731057917961; duration=0sec 2024-11-08T09:25:18,420 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:18,420 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:C 2024-11-08T09:25:18,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742318_1494 (size=12001) 2024-11-08T09:25:18,422 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/67e2352ced37455db36f5042db122d42 2024-11-08T09:25:18,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/659a0828080c4ed5a82808c0aa80de07 is 50, key is test_row_0/C:col10/1731057916744/Put/seqid=0 2024-11-08T09:25:18,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742319_1495 (size=12001) 2024-11-08T09:25:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T09:25:18,798 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/9ff818e3e5cf41da8c098de9bdf1ee73 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ff818e3e5cf41da8c098de9bdf1ee73 2024-11-08T09:25:18,801 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/A of 8144a32b044f8a5dee9d583d8360a12c into 9ff818e3e5cf41da8c098de9bdf1ee73(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:18,801 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:18,801 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/A, priority=13, startTime=1731057917961; duration=0sec 2024-11-08T09:25:18,801 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:18,801 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:A 2024-11-08T09:25:18,841 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/659a0828080c4ed5a82808c0aa80de07 2024-11-08T09:25:18,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/9ccbe47603834a1886139b65ec823cd2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ccbe47603834a1886139b65ec823cd2 2024-11-08T09:25:18,849 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ccbe47603834a1886139b65ec823cd2, entries=150, sequenceid=78, filesize=30.2 K 2024-11-08T09:25:18,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/67e2352ced37455db36f5042db122d42 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/67e2352ced37455db36f5042db122d42 2024-11-08T09:25:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,853 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/67e2352ced37455db36f5042db122d42, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T09:25:18,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/659a0828080c4ed5a82808c0aa80de07 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/659a0828080c4ed5a82808c0aa80de07 2024-11-08T09:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,856 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/659a0828080c4ed5a82808c0aa80de07, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,857 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 8144a32b044f8a5dee9d583d8360a12c in 880ms, sequenceid=78, compaction requested=false 2024-11-08T09:25:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-08T09:25:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-08T09:25:18,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2560 sec 2024-11-08T09:25:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.2600 sec 2024-11-08T09:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:18,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:25:18,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:25:18,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089a8cb55147714212a36094020a4b3dc2_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057918896/Put/seqid=0 2024-11-08T09:25:18,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742320_1496 (size=14594) 2024-11-08T09:25:18,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:18,938 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089a8cb55147714212a36094020a4b3dc2_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a8cb55147714212a36094020a4b3dc2_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:18,939 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/969fceb13494435fb3e543033d87bcad, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:18,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/969fceb13494435fb3e543033d87bcad is 175, key is test_row_0/A:col10/1731057918896/Put/seqid=0 2024-11-08T09:25:18,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742321_1497 (size=39549) 2024-11-08T09:25:18,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057978969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057978970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:18,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057978970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057978974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057978975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057979081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057979081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057979082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057979082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057979082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057979291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057979291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057979291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057979292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057979292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,347 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/969fceb13494435fb3e543033d87bcad 2024-11-08T09:25:19,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/bcaec9e1846e484eb564f3823cd9679c is 50, key is test_row_0/B:col10/1731057918896/Put/seqid=0 2024-11-08T09:25:19,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742322_1498 (size=12001) 2024-11-08T09:25:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057979601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057979601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057979601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057979601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057979601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:19,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/bcaec9e1846e484eb564f3823cd9679c 2024-11-08T09:25:19,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/f51d70e1402a4d8985d6e18a6f58b979 is 50, key is test_row_0/C:col10/1731057918896/Put/seqid=0 2024-11-08T09:25:19,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742323_1499 (size=12001) 2024-11-08T09:25:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057980111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057980111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057980112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057980112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057980112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:20,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/f51d70e1402a4d8985d6e18a6f58b979 2024-11-08T09:25:20,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/969fceb13494435fb3e543033d87bcad as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/969fceb13494435fb3e543033d87bcad 2024-11-08T09:25:20,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/969fceb13494435fb3e543033d87bcad, entries=200, sequenceid=93, filesize=38.6 K 2024-11-08T09:25:20,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/bcaec9e1846e484eb564f3823cd9679c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/bcaec9e1846e484eb564f3823cd9679c 2024-11-08T09:25:20,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/bcaec9e1846e484eb564f3823cd9679c, entries=150, sequenceid=93, filesize=11.7 K 2024-11-08T09:25:20,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/f51d70e1402a4d8985d6e18a6f58b979 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/f51d70e1402a4d8985d6e18a6f58b979 2024-11-08T09:25:20,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/f51d70e1402a4d8985d6e18a6f58b979, entries=150, sequenceid=93, filesize=11.7 K 2024-11-08T09:25:20,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8144a32b044f8a5dee9d583d8360a12c in 1278ms, sequenceid=93, compaction requested=true 2024-11-08T09:25:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:25:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:20,187 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:25:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:20,187 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:25:20,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:20,188 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:20,188 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:20,188 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/A is initiating minor compaction (all files) 2024-11-08T09:25:20,188 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/B is initiating minor compaction (all files) 2024-11-08T09:25:20,188 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/A in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:20,188 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/B in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:20,188 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ff818e3e5cf41da8c098de9bdf1ee73, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ccbe47603834a1886139b65ec823cd2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/969fceb13494435fb3e543033d87bcad] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=99.2 K 2024-11-08T09:25:20,189 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/eebd6941daf04c208af3b7121eb2cef3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/67e2352ced37455db36f5042db122d42, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/bcaec9e1846e484eb564f3823cd9679c] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=35.3 K 2024-11-08T09:25:20,189 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:20,189 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ff818e3e5cf41da8c098de9bdf1ee73, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ccbe47603834a1886139b65ec823cd2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/969fceb13494435fb3e543033d87bcad] 2024-11-08T09:25:20,189 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting eebd6941daf04c208af3b7121eb2cef3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057914676 2024-11-08T09:25:20,189 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff818e3e5cf41da8c098de9bdf1ee73, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057914676 2024-11-08T09:25:20,189 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e2352ced37455db36f5042db122d42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057916736 2024-11-08T09:25:20,189 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ccbe47603834a1886139b65ec823cd2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057916736 2024-11-08T09:25:20,189 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting bcaec9e1846e484eb564f3823cd9679c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731057918896 2024-11-08T09:25:20,189 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 969fceb13494435fb3e543033d87bcad, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731057918896 2024-11-08T09:25:20,194 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:20,195 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108c6f116010f0d4b0f94b8d5ed4246b7b3_8144a32b044f8a5dee9d583d8360a12c store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:20,195 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:20,196 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/23e30aadf556429296b1537e5efaecba is 50, key is test_row_0/B:col10/1731057918896/Put/seqid=0 2024-11-08T09:25:20,197 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108c6f116010f0d4b0f94b8d5ed4246b7b3_8144a32b044f8a5dee9d583d8360a12c, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:20,197 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c6f116010f0d4b0f94b8d5ed4246b7b3_8144a32b044f8a5dee9d583d8360a12c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:20,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742324_1500 (size=12207) 2024-11-08T09:25:20,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742325_1501 (size=4469) 2024-11-08T09:25:20,206 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/23e30aadf556429296b1537e5efaecba as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/23e30aadf556429296b1537e5efaecba 2024-11-08T09:25:20,209 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/B of 8144a32b044f8a5dee9d583d8360a12c into 23e30aadf556429296b1537e5efaecba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:20,210 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:20,210 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/B, priority=13, startTime=1731057920187; duration=0sec 2024-11-08T09:25:20,210 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:20,210 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:B 2024-11-08T09:25:20,210 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:20,211 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:20,211 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/C is initiating minor compaction (all files) 2024-11-08T09:25:20,211 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/C in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:20,211 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/00ee38fd5a4d429db03b69a3af396267, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/659a0828080c4ed5a82808c0aa80de07, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/f51d70e1402a4d8985d6e18a6f58b979] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=35.3 K 2024-11-08T09:25:20,211 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ee38fd5a4d429db03b69a3af396267, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731057914676 2024-11-08T09:25:20,211 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 659a0828080c4ed5a82808c0aa80de07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731057916736 2024-11-08T09:25:20,212 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f51d70e1402a4d8985d6e18a6f58b979, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731057918896 2024-11-08T09:25:20,218 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:20,218 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/1c4c0eb6101f475fbee8b40ce560f9d6 is 50, key is test_row_0/C:col10/1731057918896/Put/seqid=0 2024-11-08T09:25:20,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742326_1502 (size=12207) 2024-11-08T09:25:20,227 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/1c4c0eb6101f475fbee8b40ce560f9d6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1c4c0eb6101f475fbee8b40ce560f9d6 2024-11-08T09:25:20,231 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/C of 8144a32b044f8a5dee9d583d8360a12c into 1c4c0eb6101f475fbee8b40ce560f9d6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:20,231 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:20,231 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/C, priority=13, startTime=1731057920187; duration=0sec 2024-11-08T09:25:20,231 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:20,231 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:C 2024-11-08T09:25:20,604 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#A#compaction#417 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:20,604 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/3e7cfb57419c40a796d6b9f9c28a8e17 is 175, key is test_row_0/A:col10/1731057918896/Put/seqid=0 2024-11-08T09:25:20,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742327_1503 (size=31161) 2024-11-08T09:25:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T09:25:20,762 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:20,762 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-08T09:25:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T09:25:20,764 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:20,764 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:20,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T09:25:20,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T09:25:20,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:20,916 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:25:20,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088f015a62855545e79729924e1449dfb2_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057918941/Put/seqid=0 2024-11-08T09:25:20,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742328_1504 (size=12154) 2024-11-08T09:25:21,011 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/3e7cfb57419c40a796d6b9f9c28a8e17 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/3e7cfb57419c40a796d6b9f9c28a8e17 2024-11-08T09:25:21,014 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/A of 8144a32b044f8a5dee9d583d8360a12c into 3e7cfb57419c40a796d6b9f9c28a8e17(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:21,015 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:21,015 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/A, priority=13, startTime=1731057920187; duration=0sec 2024-11-08T09:25:21,015 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:21,015 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:A 2024-11-08T09:25:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T09:25:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:21,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057981134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057981135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057981136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057981137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057981137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057981241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057981241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057981241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057981252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057981252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:21,328 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088f015a62855545e79729924e1449dfb2_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088f015a62855545e79729924e1449dfb2_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/be7ae62b093042a286b3d87a5189ced0, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/be7ae62b093042a286b3d87a5189ced0 is 175, key is test_row_0/A:col10/1731057918941/Put/seqid=0 2024-11-08T09:25:21,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742329_1505 (size=30955) 2024-11-08T09:25:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T09:25:21,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057981452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057981452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057981452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057981461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057981461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,733 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/be7ae62b093042a286b3d87a5189ced0 2024-11-08T09:25:21,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/635da8b089f94bf98e06cc5deedc1880 is 50, key is test_row_0/B:col10/1731057918941/Put/seqid=0 2024-11-08T09:25:21,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742330_1506 (size=12001) 2024-11-08T09:25:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057981761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057981761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057981761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057981771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057981772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T09:25:22,142 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/635da8b089f94bf98e06cc5deedc1880 2024-11-08T09:25:22,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057982271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:22,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057982271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:22,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057982271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:22,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057982281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057982281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:22,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/9b7e9042355446218cf7986d62e4bf6b is 50, key is test_row_0/C:col10/1731057918941/Put/seqid=0 2024-11-08T09:25:22,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742331_1507 (size=12001) 2024-11-08T09:25:22,710 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/9b7e9042355446218cf7986d62e4bf6b 2024-11-08T09:25:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/be7ae62b093042a286b3d87a5189ced0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/be7ae62b093042a286b3d87a5189ced0 2024-11-08T09:25:22,717 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/be7ae62b093042a286b3d87a5189ced0, entries=150, sequenceid=119, filesize=30.2 K 2024-11-08T09:25:22,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/635da8b089f94bf98e06cc5deedc1880 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/635da8b089f94bf98e06cc5deedc1880 2024-11-08T09:25:22,721 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/635da8b089f94bf98e06cc5deedc1880, entries=150, sequenceid=119, filesize=11.7 K 2024-11-08T09:25:22,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/9b7e9042355446218cf7986d62e4bf6b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/9b7e9042355446218cf7986d62e4bf6b 2024-11-08T09:25:22,725 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/9b7e9042355446218cf7986d62e4bf6b, entries=150, sequenceid=119, filesize=11.7 K 2024-11-08T09:25:22,726 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 8144a32b044f8a5dee9d583d8360a12c in 1809ms, sequenceid=119, compaction requested=false 2024-11-08T09:25:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-08T09:25:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-08T09:25:22,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-08T09:25:22,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-11-08T09:25:22,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.9670 sec 2024-11-08T09:25:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T09:25:22,911 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:22,911 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-08T09:25:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T09:25:22,913 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:22,914 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:22,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T09:25:23,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-08T09:25:23,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:23,066 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:25:23,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a2e8ecc3799844b7a6c8ff1b6585717f_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057921136/Put/seqid=0 2024-11-08T09:25:23,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742332_1508 (size=12304) 2024-11-08T09:25:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T09:25:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:23,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:23,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057983305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057983306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057983307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057983308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057983309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057983411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057983421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057983422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057983422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057983422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:23,478 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a2e8ecc3799844b7a6c8ff1b6585717f_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a2e8ecc3799844b7a6c8ff1b6585717f_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:23,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7e6851c123746268547c35fa6c9f46a, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:23,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7e6851c123746268547c35fa6c9f46a is 175, key is test_row_0/A:col10/1731057921136/Put/seqid=0 2024-11-08T09:25:23,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742333_1509 (size=31105) 2024-11-08T09:25:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T09:25:23,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057983621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057983632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057983632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057983632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057983632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,883 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7e6851c123746268547c35fa6c9f46a 2024-11-08T09:25:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/dcdcf79d153149b99e0bcc1da838b104 is 50, key is test_row_0/B:col10/1731057921136/Put/seqid=0 2024-11-08T09:25:23,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742334_1510 (size=12151) 2024-11-08T09:25:23,893 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/dcdcf79d153149b99e0bcc1da838b104 2024-11-08T09:25:23,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/138dff9746e5427794cd9bd81076bce4 is 50, key is test_row_0/C:col10/1731057921136/Put/seqid=0 2024-11-08T09:25:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742335_1511 (size=12151) 2024-11-08T09:25:23,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057983931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057983941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057983941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057983942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:23,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057983942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T09:25:24,306 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/138dff9746e5427794cd9bd81076bce4 2024-11-08T09:25:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7e6851c123746268547c35fa6c9f46a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7e6851c123746268547c35fa6c9f46a 2024-11-08T09:25:24,312 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7e6851c123746268547c35fa6c9f46a, entries=150, sequenceid=133, filesize=30.4 K 2024-11-08T09:25:24,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/dcdcf79d153149b99e0bcc1da838b104 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/dcdcf79d153149b99e0bcc1da838b104 2024-11-08T09:25:24,316 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/dcdcf79d153149b99e0bcc1da838b104, entries=150, sequenceid=133, filesize=11.9 K 2024-11-08T09:25:24,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/138dff9746e5427794cd9bd81076bce4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/138dff9746e5427794cd9bd81076bce4 2024-11-08T09:25:24,320 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/138dff9746e5427794cd9bd81076bce4, entries=150, sequenceid=133, filesize=11.9 K 2024-11-08T09:25:24,320 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8144a32b044f8a5dee9d583d8360a12c in 1255ms, sequenceid=133, compaction requested=true 2024-11-08T09:25:24,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:24,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:24,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-08T09:25:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-08T09:25:24,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-08T09:25:24,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-08T09:25:24,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4120 sec 2024-11-08T09:25:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:25:24,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:25:24,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c1c069b173bf48d6b54024cc98bc4105_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057923305/Put/seqid=0 2024-11-08T09:25:24,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742336_1512 (size=14794) 2024-11-08T09:25:24,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057984457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057984458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057984461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057984462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057984463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057984571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057984571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057984571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057984571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057984571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,654 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T09:25:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057984782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057984782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057984782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057984782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057984782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:24,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:24,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c1c069b173bf48d6b54024cc98bc4105_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c1c069b173bf48d6b54024cc98bc4105_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:24,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1ef9d49baf4341ab8d07b0188d5cc122, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:24,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1ef9d49baf4341ab8d07b0188d5cc122 is 175, key is test_row_0/A:col10/1731057923305/Put/seqid=0 2024-11-08T09:25:24,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742337_1513 (size=39749) 2024-11-08T09:25:24,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1ef9d49baf4341ab8d07b0188d5cc122 2024-11-08T09:25:24,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/5c3b612e6ad1480e956898cb9b3ff473 is 50, key is test_row_0/B:col10/1731057923305/Put/seqid=0 2024-11-08T09:25:24,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742338_1514 (size=12151) 2024-11-08T09:25:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T09:25:25,061 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:25,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-08T09:25:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T09:25:25,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:25,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:25,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:25,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057985091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057985092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057985092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057985092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057985092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T09:25:25,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T09:25:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/5c3b612e6ad1480e956898cb9b3ff473 2024-11-08T09:25:25,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/edc2aefd137840f2a1944565b0ec1282 is 50, key is test_row_0/C:col10/1731057923305/Put/seqid=0 2024-11-08T09:25:25,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742339_1515 (size=12151) 2024-11-08T09:25:25,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T09:25:25,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:25,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T09:25:25,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T09:25:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057985601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057985601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057985601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057985601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057985601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:25,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T09:25:25,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:25,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T09:25:25,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/edc2aefd137840f2a1944565b0ec1282 2024-11-08T09:25:25,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1ef9d49baf4341ab8d07b0188d5cc122 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1ef9d49baf4341ab8d07b0188d5cc122 2024-11-08T09:25:25,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1ef9d49baf4341ab8d07b0188d5cc122, entries=200, sequenceid=157, filesize=38.8 K 2024-11-08T09:25:25,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/5c3b612e6ad1480e956898cb9b3ff473 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/5c3b612e6ad1480e956898cb9b3ff473 2024-11-08T09:25:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/5c3b612e6ad1480e956898cb9b3ff473, entries=150, sequenceid=157, filesize=11.9 K 2024-11-08T09:25:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/edc2aefd137840f2a1944565b0ec1282 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/edc2aefd137840f2a1944565b0ec1282 2024-11-08T09:25:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/edc2aefd137840f2a1944565b0ec1282, entries=150, sequenceid=157, filesize=11.9 K 2024-11-08T09:25:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 8144a32b044f8a5dee9d583d8360a12c in 1271ms, sequenceid=157, compaction requested=true 2024-11-08T09:25:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:25:25,716 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:25:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:25:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,716 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:25:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:25:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,717 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:25:25,717 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/A is initiating minor compaction (all files) 2024-11-08T09:25:25,717 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/A in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,717 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/3e7cfb57419c40a796d6b9f9c28a8e17, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/be7ae62b093042a286b3d87a5189ced0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7e6851c123746268547c35fa6c9f46a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1ef9d49baf4341ab8d07b0188d5cc122] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=129.9 K 2024-11-08T09:25:25,717 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,717 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/3e7cfb57419c40a796d6b9f9c28a8e17, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/be7ae62b093042a286b3d87a5189ced0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7e6851c123746268547c35fa6c9f46a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1ef9d49baf4341ab8d07b0188d5cc122] 2024-11-08T09:25:25,718 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:25:25,718 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/B is initiating minor compaction (all files) 2024-11-08T09:25:25,718 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/B in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,718 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/23e30aadf556429296b1537e5efaecba, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/635da8b089f94bf98e06cc5deedc1880, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/dcdcf79d153149b99e0bcc1da838b104, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/5c3b612e6ad1480e956898cb9b3ff473] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=47.4 K 2024-11-08T09:25:25,718 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e30aadf556429296b1537e5efaecba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731057918896 2024-11-08T09:25:25,718 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e7cfb57419c40a796d6b9f9c28a8e17, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731057918896 2024-11-08T09:25:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,718 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 635da8b089f94bf98e06cc5deedc1880, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731057918941 2024-11-08T09:25:25,718 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7ae62b093042a286b3d87a5189ced0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731057918941 2024-11-08T09:25:25,719 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e6851c123746268547c35fa6c9f46a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057921135 2024-11-08T09:25:25,719 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting dcdcf79d153149b99e0bcc1da838b104, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057921135 2024-11-08T09:25:25,719 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef9d49baf4341ab8d07b0188d5cc122, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731057923305 2024-11-08T09:25:25,719 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3b612e6ad1480e956898cb9b3ff473, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731057923305 2024-11-08T09:25:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,724 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,726 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,726 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/f895539e661a4736a33b179168bfd83c is 50, key is test_row_0/B:col10/1731057923305/Put/seqid=0 2024-11-08T09:25:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,728 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110861f8382313d04895a5e88123c045a2af_8144a32b044f8a5dee9d583d8360a12c store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,731 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110861f8382313d04895a5e88123c045a2af_8144a32b044f8a5dee9d583d8360a12c, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:25,731 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110861f8382313d04895a5e88123c045a2af_8144a32b044f8a5dee9d583d8360a12c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742340_1516 (size=12493) 2024-11-08T09:25:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742341_1517 (size=4469) 2024-11-08T09:25:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T09:25:25,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:25,826 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T09:25:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110861d53da8c52a4adba27e4c0f477eeeca_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057924456/Put/seqid=0 2024-11-08T09:25:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742342_1518 (size=9814) 2024-11-08T09:25:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,150 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/f895539e661a4736a33b179168bfd83c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/f895539e661a4736a33b179168bfd83c 2024-11-08T09:25:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,154 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/B of 8144a32b044f8a5dee9d583d8360a12c into f895539e661a4736a33b179168bfd83c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:26,154 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:26,154 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/B, priority=12, startTime=1731057925716; duration=0sec 2024-11-08T09:25:26,154 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:26,154 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:B 2024-11-08T09:25:26,155 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T09:25:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,157 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#A#compaction#428 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:26,157 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T09:25:26,157 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/C is initiating minor compaction (all files) 2024-11-08T09:25:26,157 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/C in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:26,157 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1c4c0eb6101f475fbee8b40ce560f9d6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/9b7e9042355446218cf7986d62e4bf6b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/138dff9746e5427794cd9bd81076bce4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/edc2aefd137840f2a1944565b0ec1282] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=47.4 K 2024-11-08T09:25:26,157 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f91d9125ed8a477c980567bb54d1b04d is 175, key is test_row_0/A:col10/1731057923305/Put/seqid=0 2024-11-08T09:25:26,158 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c4c0eb6101f475fbee8b40ce560f9d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731057918896 2024-11-08T09:25:26,158 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b7e9042355446218cf7986d62e4bf6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731057918941 2024-11-08T09:25:26,159 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 138dff9746e5427794cd9bd81076bce4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731057921135 2024-11-08T09:25:26,159 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting edc2aefd137840f2a1944565b0ec1282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731057923305 2024-11-08T09:25:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742343_1519 (size=31447) 2024-11-08T09:25:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,167 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f91d9125ed8a477c980567bb54d1b04d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f91d9125ed8a477c980567bb54d1b04d 2024-11-08T09:25:26,169 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:26,170 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/e6319fae0cd14b588eb52168afb7b7fc is 50, key is test_row_0/C:col10/1731057923305/Put/seqid=0 2024-11-08T09:25:26,172 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/A of 8144a32b044f8a5dee9d583d8360a12c into f91d9125ed8a477c980567bb54d1b04d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:26,172 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:26,172 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/A, priority=12, startTime=1731057925716; duration=0sec 2024-11-08T09:25:26,172 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:26,172 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:A 2024-11-08T09:25:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742344_1520 (size=12493) 2024-11-08T09:25:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T09:25:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,241 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110861d53da8c52a4adba27e4c0f477eeeca_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110861d53da8c52a4adba27e4c0f477eeeca_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/ddc6e0e647e241c184c647b8356d486c, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/ddc6e0e647e241c184c647b8356d486c is 175, key is test_row_0/A:col10/1731057924456/Put/seqid=0 2024-11-08T09:25:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742345_1521 (size=22461) 2024-11-08T09:25:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,258 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/ddc6e0e647e241c184c647b8356d486c 2024-11-08T09:25:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/57bc4e4982784b198d64b499108e140b is 50, key is test_row_0/B:col10/1731057924456/Put/seqid=0 2024-11-08T09:25:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742346_1522 (size=9757) 2024-11-08T09:25:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,583 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/e6319fae0cd14b588eb52168afb7b7fc as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e6319fae0cd14b588eb52168afb7b7fc 2024-11-08T09:25:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,588 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/C of 8144a32b044f8a5dee9d583d8360a12c into e6319fae0cd14b588eb52168afb7b7fc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:26,588 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:26,588 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/C, priority=12, startTime=1731057925716; duration=0sec 2024-11-08T09:25:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,588 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:26,588 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:C 2024-11-08T09:25:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:26,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057986633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057986634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057986663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057986663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057986663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,670 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/57bc4e4982784b198d64b499108e140b 2024-11-08T09:25:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/05ac276b30a04bf186f224fdac39c68d is 50, key is test_row_0/C:col10/1731057924456/Put/seqid=0 2024-11-08T09:25:26,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742347_1523 (size=9757) 2024-11-08T09:25:26,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057986771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057986771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057986771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057986771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057986772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057986981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057986982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057986982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057986982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057986982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,080 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/05ac276b30a04bf186f224fdac39c68d 2024-11-08T09:25:27,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/ddc6e0e647e241c184c647b8356d486c as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/ddc6e0e647e241c184c647b8356d486c 2024-11-08T09:25:27,086 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/ddc6e0e647e241c184c647b8356d486c, entries=100, sequenceid=170, filesize=21.9 K 2024-11-08T09:25:27,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/57bc4e4982784b198d64b499108e140b as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/57bc4e4982784b198d64b499108e140b 2024-11-08T09:25:27,089 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/57bc4e4982784b198d64b499108e140b, entries=100, sequenceid=170, filesize=9.5 K 2024-11-08T09:25:27,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-08T09:25:27,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/05ac276b30a04bf186f224fdac39c68d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/05ac276b30a04bf186f224fdac39c68d 2024-11-08T09:25:27,093 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/05ac276b30a04bf186f224fdac39c68d, entries=100, sequenceid=170, filesize=9.5 K 2024-11-08T09:25:27,093 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 8144a32b044f8a5dee9d583d8360a12c in 1267ms, sequenceid=170, compaction requested=false 2024-11-08T09:25:27,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-08T09:25:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-08T09:25:27,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-08T09:25:27,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0310 sec 2024-11-08T09:25:27,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.0350 sec 2024-11-08T09:25:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T09:25:27,212 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:27,212 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-08T09:25:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T09:25:27,214 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:27,214 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:27,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:27,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:25:27,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T09:25:27,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083a4dc26e263a4fe998d32e4ae289942d_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057927293/Put/seqid=0 2024-11-08T09:25:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057987296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057987299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742348_1524 (size=12304) 2024-11-08T09:25:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057987300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057987301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057987302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T09:25:27,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:27,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:27,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057987412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057987412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057987412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057987412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057987412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T09:25:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057987621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057987621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057987621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057987621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057987621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,704 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:27,707 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083a4dc26e263a4fe998d32e4ae289942d_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083a4dc26e263a4fe998d32e4ae289942d_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:27,707 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1469d86f5c404d2bb4828f7b4ee7e508, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:27,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1469d86f5c404d2bb4828f7b4ee7e508 is 175, key is test_row_0/A:col10/1731057927293/Put/seqid=0 2024-11-08T09:25:27,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742349_1525 (size=31105) 2024-11-08T09:25:27,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:27,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:27,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T09:25:27,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057987932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057987932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057987932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057987932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057987932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:27,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:27,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,111 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1469d86f5c404d2bb4828f7b4ee7e508 2024-11-08T09:25:28,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/6ba90239c1314177b9b2d3ef66ba926e is 50, key is test_row_0/B:col10/1731057927293/Put/seqid=0 2024-11-08T09:25:28,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742350_1526 (size=12151) 2024-11-08T09:25:28,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/6ba90239c1314177b9b2d3ef66ba926e 2024-11-08T09:25:28,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/e8b94692b3114f6ba03c067c2111dd6f is 50, key is test_row_0/C:col10/1731057927293/Put/seqid=0 2024-11-08T09:25:28,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742351_1527 (size=12151) 2024-11-08T09:25:28,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T09:25:28,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:28,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057988441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057988441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057988441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057988441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:28,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057988441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:28,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/e8b94692b3114f6ba03c067c2111dd6f 2024-11-08T09:25:28,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/1469d86f5c404d2bb4828f7b4ee7e508 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1469d86f5c404d2bb4828f7b4ee7e508 2024-11-08T09:25:28,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1469d86f5c404d2bb4828f7b4ee7e508, entries=150, sequenceid=199, filesize=30.4 K 2024-11-08T09:25:28,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/6ba90239c1314177b9b2d3ef66ba926e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6ba90239c1314177b9b2d3ef66ba926e 2024-11-08T09:25:28,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6ba90239c1314177b9b2d3ef66ba926e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-08T09:25:28,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/e8b94692b3114f6ba03c067c2111dd6f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e8b94692b3114f6ba03c067c2111dd6f 2024-11-08T09:25:28,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e8b94692b3114f6ba03c067c2111dd6f, entries=150, sequenceid=199, filesize=11.9 K 2024-11-08T09:25:28,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8144a32b044f8a5dee9d583d8360a12c in 1247ms, sequenceid=199, compaction requested=true 2024-11-08T09:25:28,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:28,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:25:28,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:28,541 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:28,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:25:28,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:28,541 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:28,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:25:28,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:28,542 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:28,542 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:28,542 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/B is initiating minor compaction (all files) 2024-11-08T09:25:28,542 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/A is initiating minor compaction (all files) 2024-11-08T09:25:28,542 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/A in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,542 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/B in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,542 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/f895539e661a4736a33b179168bfd83c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/57bc4e4982784b198d64b499108e140b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6ba90239c1314177b9b2d3ef66ba926e] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=33.6 K 2024-11-08T09:25:28,542 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f91d9125ed8a477c980567bb54d1b04d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/ddc6e0e647e241c184c647b8356d486c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1469d86f5c404d2bb4828f7b4ee7e508] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=83.0 K 2024-11-08T09:25:28,542 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,542 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f91d9125ed8a477c980567bb54d1b04d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/ddc6e0e647e241c184c647b8356d486c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1469d86f5c404d2bb4828f7b4ee7e508] 2024-11-08T09:25:28,542 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f91d9125ed8a477c980567bb54d1b04d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731057923305 2024-11-08T09:25:28,542 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting f895539e661a4736a33b179168bfd83c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731057923305 2024-11-08T09:25:28,543 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 57bc4e4982784b198d64b499108e140b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731057924456 2024-11-08T09:25:28,543 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc6e0e647e241c184c647b8356d486c, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731057924456 2024-11-08T09:25:28,543 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba90239c1314177b9b2d3ef66ba926e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057926639 2024-11-08T09:25:28,543 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1469d86f5c404d2bb4828f7b4ee7e508, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057926639 2024-11-08T09:25:28,548 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:28,549 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/676ad635740941a999df4de3c8df6cc6 is 50, key is test_row_0/B:col10/1731057927293/Put/seqid=0 2024-11-08T09:25:28,557 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:28,569 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108d4a6865adf1e4f6aa458c89e0e49cb9e_8144a32b044f8a5dee9d583d8360a12c store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:28,570 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108d4a6865adf1e4f6aa458c89e0e49cb9e_8144a32b044f8a5dee9d583d8360a12c, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:28,571 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d4a6865adf1e4f6aa458c89e0e49cb9e_8144a32b044f8a5dee9d583d8360a12c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:28,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742352_1528 (size=12595) 2024-11-08T09:25:28,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T09:25:28,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,587 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T09:25:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742353_1529 (size=4469) 2024-11-08T09:25:28,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108631cbe61c1d8460da5272ed50dadfcbe_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057927300/Put/seqid=0 2024-11-08T09:25:28,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742354_1530 (size=12304) 2024-11-08T09:25:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:28,623 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108631cbe61c1d8460da5272ed50dadfcbe_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108631cbe61c1d8460da5272ed50dadfcbe_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/b701aac3be7a4142bf243f6c4a9736b0, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:28,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/b701aac3be7a4142bf243f6c4a9736b0 is 175, key is test_row_0/A:col10/1731057927300/Put/seqid=0 2024-11-08T09:25:28,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742355_1531 (size=31105) 2024-11-08T09:25:28,630 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/b701aac3be7a4142bf243f6c4a9736b0 2024-11-08T09:25:28,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/2b86a87c698649ba96aa8825795f522d is 50, key is test_row_0/B:col10/1731057927300/Put/seqid=0 2024-11-08T09:25:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742356_1532 (size=12151) 2024-11-08T09:25:28,982 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/676ad635740941a999df4de3c8df6cc6 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/676ad635740941a999df4de3c8df6cc6 2024-11-08T09:25:28,987 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/B of 8144a32b044f8a5dee9d583d8360a12c into 676ad635740941a999df4de3c8df6cc6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:28,987 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:28,987 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/B, priority=13, startTime=1731057928541; duration=0sec 2024-11-08T09:25:28,987 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:28,987 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:B 2024-11-08T09:25:28,987 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:28,988 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:28,988 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/C is initiating minor compaction (all files) 2024-11-08T09:25:28,988 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/C in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:28,988 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e6319fae0cd14b588eb52168afb7b7fc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/05ac276b30a04bf186f224fdac39c68d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e8b94692b3114f6ba03c067c2111dd6f] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=33.6 K 2024-11-08T09:25:28,989 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e6319fae0cd14b588eb52168afb7b7fc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731057923305 2024-11-08T09:25:28,989 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ac276b30a04bf186f224fdac39c68d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731057924456 2024-11-08T09:25:28,989 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b94692b3114f6ba03c067c2111dd6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057926639 2024-11-08T09:25:28,995 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:28,995 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/af83eddfcf5f4bebbbca6b1877972c27 is 50, key is test_row_0/C:col10/1731057927293/Put/seqid=0 2024-11-08T09:25:28,999 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#A#compaction#438 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:28,999 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c3d2aff571ac4d23852e04531597c62f is 175, key is test_row_0/A:col10/1731057927293/Put/seqid=0 2024-11-08T09:25:29,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742357_1533 (size=12595) 2024-11-08T09:25:29,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742358_1534 (size=31549) 2024-11-08T09:25:29,047 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/2b86a87c698649ba96aa8825795f522d 2024-11-08T09:25:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/5c382527c71046738c2d7ba56a98185d is 50, key is test_row_0/C:col10/1731057927300/Put/seqid=0 2024-11-08T09:25:29,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742359_1535 (size=12151) 2024-11-08T09:25:29,072 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/5c382527c71046738c2d7ba56a98185d 2024-11-08T09:25:29,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/b701aac3be7a4142bf243f6c4a9736b0 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/b701aac3be7a4142bf243f6c4a9736b0 2024-11-08T09:25:29,082 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/b701aac3be7a4142bf243f6c4a9736b0, entries=150, sequenceid=209, filesize=30.4 K 2024-11-08T09:25:29,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/2b86a87c698649ba96aa8825795f522d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/2b86a87c698649ba96aa8825795f522d 2024-11-08T09:25:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,086 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/2b86a87c698649ba96aa8825795f522d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-08T09:25:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/5c382527c71046738c2d7ba56a98185d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/5c382527c71046738c2d7ba56a98185d 2024-11-08T09:25:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,093 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/5c382527c71046738c2d7ba56a98185d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-08T09:25:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,094 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 8144a32b044f8a5dee9d583d8360a12c in 507ms, sequenceid=209, compaction requested=false 2024-11-08T09:25:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:29,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:29,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-08T09:25:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-08T09:25:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-08T09:25:29,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-08T09:25:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8910 sec 2024-11-08T09:25:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T09:25:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,362 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,363 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-08T09:25:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,367 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T09:25:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,367 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:29,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,416 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/af83eddfcf5f4bebbbca6b1877972c27 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/af83eddfcf5f4bebbbca6b1877972c27 2024-11-08T09:25:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,420 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c3d2aff571ac4d23852e04531597c62f as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c3d2aff571ac4d23852e04531597c62f 2024-11-08T09:25:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,422 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/C of 8144a32b044f8a5dee9d583d8360a12c into af83eddfcf5f4bebbbca6b1877972c27(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:29,422 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:29,422 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/C, priority=13, startTime=1731057928541; duration=0sec 2024-11-08T09:25:29,422 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:29,422 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:C 2024-11-08T09:25:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,426 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/A of 8144a32b044f8a5dee9d583d8360a12c into c3d2aff571ac4d23852e04531597c62f(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,426 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:29,426 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/A, priority=13, startTime=1731057928541; duration=0sec 2024-11-08T09:25:29,426 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:29,426 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:A 2024-11-08T09:25:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T09:25:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:29,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:25:29,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T09:25:29,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110899c6e6c1254647d08186a4fa1d9580b2_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057929487/Put/seqid=0 2024-11-08T09:25:29,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057989512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T09:25:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057989515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:29,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057989516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T09:25:29,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057989519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057989519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742360_1536 (size=14794) 2024-11-08T09:25:29,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:29,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110899c6e6c1254647d08186a4fa1d9580b2_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110899c6e6c1254647d08186a4fa1d9580b2_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:29,531 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f0b61d85219242589a051b0791c8dafd, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:29,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f0b61d85219242589a051b0791c8dafd is 175, key is test_row_0/A:col10/1731057929487/Put/seqid=0 2024-11-08T09:25:29,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742361_1537 (size=39749) 2024-11-08T09:25:29,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f0b61d85219242589a051b0791c8dafd 2024-11-08T09:25:29,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/591afe19d00c4500aff79004d4620471 is 50, key is test_row_0/B:col10/1731057929487/Put/seqid=0 2024-11-08T09:25:29,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742362_1538 (size=12151) 2024-11-08T09:25:29,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/591afe19d00c4500aff79004d4620471 2024-11-08T09:25:29,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/b674d029570b4e0ebd24902ed2556115 is 50, key is test_row_0/C:col10/1731057929487/Put/seqid=0 2024-11-08T09:25:29,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742363_1539 (size=12151) 2024-11-08T09:25:29,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/b674d029570b4e0ebd24902ed2556115 2024-11-08T09:25:29,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057989621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/f0b61d85219242589a051b0791c8dafd as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f0b61d85219242589a051b0791c8dafd 2024-11-08T09:25:29,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057989632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057989632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057989632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057989632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f0b61d85219242589a051b0791c8dafd, entries=200, sequenceid=225, filesize=38.8 K 2024-11-08T09:25:29,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/591afe19d00c4500aff79004d4620471 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/591afe19d00c4500aff79004d4620471 2024-11-08T09:25:29,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/591afe19d00c4500aff79004d4620471, entries=150, sequenceid=225, filesize=11.9 K 2024-11-08T09:25:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/b674d029570b4e0ebd24902ed2556115 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/b674d029570b4e0ebd24902ed2556115 2024-11-08T09:25:29,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/b674d029570b4e0ebd24902ed2556115, entries=150, sequenceid=225, filesize=11.9 K 2024-11-08T09:25:29,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 8144a32b044f8a5dee9d583d8360a12c in 159ms, sequenceid=225, compaction requested=true 2024-11-08T09:25:29,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:29,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:25:29,647 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:29,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:29,647 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:29,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:25:29,648 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:29,648 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/A is initiating minor compaction (all files) 2024-11-08T09:25:29,648 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/A in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:29,648 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c3d2aff571ac4d23852e04531597c62f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/b701aac3be7a4142bf243f6c4a9736b0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f0b61d85219242589a051b0791c8dafd] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=100.0 K 2024-11-08T09:25:29,648 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:29,648 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. files: [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c3d2aff571ac4d23852e04531597c62f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/b701aac3be7a4142bf243f6c4a9736b0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f0b61d85219242589a051b0791c8dafd] 2024-11-08T09:25:29,648 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:29,648 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/B is initiating minor compaction (all files) 2024-11-08T09:25:29,649 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/B in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:29,649 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/676ad635740941a999df4de3c8df6cc6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/2b86a87c698649ba96aa8825795f522d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/591afe19d00c4500aff79004d4620471] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=36.0 K 2024-11-08T09:25:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T09:25:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:29,650 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d2aff571ac4d23852e04531597c62f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057926639 2024-11-08T09:25:29,651 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 676ad635740941a999df4de3c8df6cc6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057926639 2024-11-08T09:25:29,652 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting b701aac3be7a4142bf243f6c4a9736b0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731057927299 2024-11-08T09:25:29,652 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b86a87c698649ba96aa8825795f522d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731057927299 2024-11-08T09:25:29,652 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0b61d85219242589a051b0791c8dafd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731057929479 2024-11-08T09:25:29,652 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 591afe19d00c4500aff79004d4620471, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731057929481 2024-11-08T09:25:29,659 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:29,661 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108ae6cb95ff6384827b0484f302153e760_8144a32b044f8a5dee9d583d8360a12c store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:29,661 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:29,661 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/74d1654e7f784fca8aeecacb013c8dde is 50, key is test_row_0/B:col10/1731057929487/Put/seqid=0 2024-11-08T09:25:29,663 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108ae6cb95ff6384827b0484f302153e760_8144a32b044f8a5dee9d583d8360a12c, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:29,663 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ae6cb95ff6384827b0484f302153e760_8144a32b044f8a5dee9d583d8360a12c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:29,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T09:25:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:29,673 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T09:25:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T09:25:29,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742364_1540 (size=12697) 2024-11-08T09:25:29,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742365_1541 (size=4469) 2024-11-08T09:25:29,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b33858b8d04e407392459b5a95989263_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057929516/Put/seqid=0 2024-11-08T09:25:29,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742366_1542 (size=12304) 2024-11-08T09:25:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:29,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:29,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057989845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057989845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057989845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057989846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057989849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057989951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057989961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057989962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057989962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057989962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T09:25:30,084 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#A#compaction#446 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:30,084 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/5254af955ffe4d0ba9019dc13f027e5e is 175, key is test_row_0/A:col10/1731057929487/Put/seqid=0 2024-11-08T09:25:30,087 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/74d1654e7f784fca8aeecacb013c8dde as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/74d1654e7f784fca8aeecacb013c8dde 2024-11-08T09:25:30,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742367_1543 (size=31651) 2024-11-08T09:25:30,091 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/5254af955ffe4d0ba9019dc13f027e5e as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/5254af955ffe4d0ba9019dc13f027e5e 2024-11-08T09:25:30,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:30,091 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/B of 8144a32b044f8a5dee9d583d8360a12c into 74d1654e7f784fca8aeecacb013c8dde(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:30,091 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:30,091 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/B, priority=13, startTime=1731057929647; duration=0sec 2024-11-08T09:25:30,091 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T09:25:30,091 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:B 2024-11-08T09:25:30,091 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T09:25:30,092 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T09:25:30,093 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1541): 8144a32b044f8a5dee9d583d8360a12c/C is initiating minor compaction (all files) 2024-11-08T09:25:30,093 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8144a32b044f8a5dee9d583d8360a12c/C in TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:30,093 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/af83eddfcf5f4bebbbca6b1877972c27, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/5c382527c71046738c2d7ba56a98185d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/b674d029570b4e0ebd24902ed2556115] into tmpdir=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp, totalSize=36.0 K 2024-11-08T09:25:30,093 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting af83eddfcf5f4bebbbca6b1877972c27, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731057926639 2024-11-08T09:25:30,094 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c382527c71046738c2d7ba56a98185d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731057927299 2024-11-08T09:25:30,094 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] compactions.Compactor(225): Compacting b674d029570b4e0ebd24902ed2556115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731057929481 2024-11-08T09:25:30,094 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b33858b8d04e407392459b5a95989263_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b33858b8d04e407392459b5a95989263_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7c0c2c8e6a14801969091ea2e428464, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7c0c2c8e6a14801969091ea2e428464 is 175, key is test_row_0/A:col10/1731057929516/Put/seqid=0 2024-11-08T09:25:30,097 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/A of 8144a32b044f8a5dee9d583d8360a12c into 5254af955ffe4d0ba9019dc13f027e5e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:30,097 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:30,097 INFO [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/A, priority=13, startTime=1731057929647; duration=0sec 2024-11-08T09:25:30,097 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:30,097 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:A 2024-11-08T09:25:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742368_1544 (size=31105) 2024-11-08T09:25:30,102 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8144a32b044f8a5dee9d583d8360a12c#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T09:25:30,102 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/799dd11f2f20422981e2ce2e8ef6a0a1 is 50, key is test_row_0/C:col10/1731057929487/Put/seqid=0 2024-11-08T09:25:30,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742369_1545 (size=12697) 2024-11-08T09:25:30,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057990162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057990172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057990172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057990172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057990172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057990472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057990481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057990481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057990481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057990481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,499 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7c0c2c8e6a14801969091ea2e428464 2024-11-08T09:25:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T09:25:30,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/85ffe52edea54a5aaa2be201b4e424d9 is 50, key is test_row_0/B:col10/1731057929516/Put/seqid=0 2024-11-08T09:25:30,513 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/799dd11f2f20422981e2ce2e8ef6a0a1 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/799dd11f2f20422981e2ce2e8ef6a0a1 2024-11-08T09:25:30,518 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8144a32b044f8a5dee9d583d8360a12c/C of 8144a32b044f8a5dee9d583d8360a12c into 799dd11f2f20422981e2ce2e8ef6a0a1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T09:25:30,518 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:30,518 INFO [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., storeName=8144a32b044f8a5dee9d583d8360a12c/C, priority=13, startTime=1731057929650; duration=0sec 2024-11-08T09:25:30,518 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:30,518 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:C 2024-11-08T09:25:30,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742370_1546 (size=12151) 2024-11-08T09:25:30,926 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/85ffe52edea54a5aaa2be201b4e424d9 2024-11-08T09:25:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0cecc5d08d0848308b30293d1826d7d3 is 50, key is test_row_0/C:col10/1731057929516/Put/seqid=0 2024-11-08T09:25:30,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742371_1547 (size=12151) 2024-11-08T09:25:30,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057990982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057990991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057990991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057990991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057990991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:30,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:31,335 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0cecc5d08d0848308b30293d1826d7d3 2024-11-08T09:25:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/c7c0c2c8e6a14801969091ea2e428464 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7c0c2c8e6a14801969091ea2e428464 2024-11-08T09:25:31,341 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7c0c2c8e6a14801969091ea2e428464, entries=150, sequenceid=249, filesize=30.4 K 2024-11-08T09:25:31,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/85ffe52edea54a5aaa2be201b4e424d9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/85ffe52edea54a5aaa2be201b4e424d9 2024-11-08T09:25:31,344 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/85ffe52edea54a5aaa2be201b4e424d9, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T09:25:31,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0cecc5d08d0848308b30293d1826d7d3 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0cecc5d08d0848308b30293d1826d7d3 2024-11-08T09:25:31,347 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0cecc5d08d0848308b30293d1826d7d3, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T09:25:31,347 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8144a32b044f8a5dee9d583d8360a12c in 1674ms, sequenceid=249, compaction requested=false 2024-11-08T09:25:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-08T09:25:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-08T09:25:31,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-08T09:25:31,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9820 sec 2024-11-08T09:25:31,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.9880 sec 2024-11-08T09:25:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T09:25:31,511 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:31,512 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T09:25:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-08T09:25:31,513 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T09:25:31,514 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T09:25:31,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T09:25:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T09:25:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T09:25:31,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39151 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T09:25:31,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:31,666 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T09:25:31,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b65d082fe4a94cc2a859fd20c16b28ad_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057929843/Put/seqid=0 2024-11-08T09:25:31,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742372_1548 (size=12404) 2024-11-08T09:25:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T09:25:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:32,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. as already flushing 2024-11-08T09:25:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057992019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057992020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057992021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057992022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057992023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:32,077 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b65d082fe4a94cc2a859fd20c16b28ad_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b65d082fe4a94cc2a859fd20c16b28ad_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/fa3234c8ec594b048a9afa437c5e36e9, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/fa3234c8ec594b048a9afa437c5e36e9 is 175, key is test_row_0/A:col10/1731057929843/Put/seqid=0 2024-11-08T09:25:32,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742373_1549 (size=31205) 2024-11-08T09:25:32,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057992131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057992131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057992131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057992131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057992131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T09:25:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057992341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057992341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057992341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057992341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057992341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,429 INFO [Thread-2173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2173. 2024-11-08T09:25:32,429 DEBUG [Thread-2173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:32,429 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,429 DEBUG [Thread-2173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:32,429 INFO [Thread-2167 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2167. 2024-11-08T09:25:32,429 DEBUG [Thread-2167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:32,429 DEBUG [Thread-2167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,430 DEBUG [Thread-2167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:32,430 INFO [Thread-2171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2171. 2024-11-08T09:25:32,430 DEBUG [Thread-2171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:32,430 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,430 DEBUG [Thread-2171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:32,432 INFO [Thread-2169 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2169. 2024-11-08T09:25:32,432 DEBUG [Thread-2169 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:32,432 DEBUG [Thread-2169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,432 DEBUG [Thread-2169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:32,432 INFO [Thread-2165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2165. 2024-11-08T09:25:32,432 DEBUG [Thread-2165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:32,432 DEBUG [Thread-2165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,432 DEBUG [Thread-2165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:32,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:32,482 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/fa3234c8ec594b048a9afa437c5e36e9 2024-11-08T09:25:32,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/80cd97a5e1104568867b23293040ac47 is 50, key is test_row_0/B:col10/1731057929843/Put/seqid=0 2024-11-08T09:25:32,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742374_1550 (size=12251) 2024-11-08T09:25:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T09:25:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057992653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057992653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057992654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057992654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057992654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:32,899 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/80cd97a5e1104568867b23293040ac47 2024-11-08T09:25:32,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/93f496776b044944b5f29091f8ef7ba7 is 50, key is test_row_0/C:col10/1731057929843/Put/seqid=0 2024-11-08T09:25:32,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742375_1551 (size=12251) 2024-11-08T09:25:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35886 deadline: 1731057993161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35898 deadline: 1731057993162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35872 deadline: 1731057993162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:33,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T09:25:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35924 deadline: 1731057993162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35882 deadline: 1731057993163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 , the old value is region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T09:25:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., hostname=cf4d76213909,39151,1731057746690, seqNum=5 because the exception is null or not the one we care about 2024-11-08T09:25:33,319 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/93f496776b044944b5f29091f8ef7ba7 2024-11-08T09:25:33,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/fa3234c8ec594b048a9afa437c5e36e9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/fa3234c8ec594b048a9afa437c5e36e9 2024-11-08T09:25:33,330 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/fa3234c8ec594b048a9afa437c5e36e9, entries=150, sequenceid=264, filesize=30.5 K 2024-11-08T09:25:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/80cd97a5e1104568867b23293040ac47 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/80cd97a5e1104568867b23293040ac47 2024-11-08T09:25:33,335 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/80cd97a5e1104568867b23293040ac47, entries=150, sequenceid=264, filesize=12.0 K 2024-11-08T09:25:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/93f496776b044944b5f29091f8ef7ba7 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/93f496776b044944b5f29091f8ef7ba7 2024-11-08T09:25:33,339 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/93f496776b044944b5f29091f8ef7ba7, entries=150, sequenceid=264, filesize=12.0 K 2024-11-08T09:25:33,340 INFO [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8144a32b044f8a5dee9d583d8360a12c in 1674ms, sequenceid=264, compaction requested=true 2024-11-08T09:25:33,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:33,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:33,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/cf4d76213909:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-08T09:25:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-08T09:25:33,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-08T09:25:33,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8270 sec 2024-11-08T09:25:33,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8320 sec 2024-11-08T09:25:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T09:25:33,673 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39151 {}] regionserver.HRegion(8855): Flush requested on 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:34,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T09:25:34,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T09:25:34,178 INFO [Thread-2154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2154. 2024-11-08T09:25:34,178 DEBUG [Thread-2154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:34,178 DEBUG [Thread-2154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,179 INFO [Thread-2158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2158. 2024-11-08T09:25:34,179 DEBUG [Thread-2154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,179 DEBUG [Thread-2158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:34,179 DEBUG [Thread-2158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,179 DEBUG [Thread-2158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:34,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:34,183 INFO [Thread-2156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2156. 2024-11-08T09:25:34,183 INFO [Thread-2160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2160. 2024-11-08T09:25:34,183 INFO [Thread-2162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2162. 2024-11-08T09:25:34,183 DEBUG [Thread-2160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:34,183 DEBUG [Thread-2156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:34,183 DEBUG [Thread-2162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T09:25:34,183 DEBUG [Thread-2160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,183 DEBUG [Thread-2156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,183 DEBUG [Thread-2162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,183 DEBUG [Thread-2156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,183 DEBUG [Thread-2160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,183 DEBUG [Thread-2162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:34,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:34,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4477 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4285 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4413 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4469 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4297 2024-11-08T09:25:34,183 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T09:25:34,184 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:25:34,184 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:25:34,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:34,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:34,184 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T09:25:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T09:25:34,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108614ac0c28ca4485b8f731b80ef762a97_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057932020/Put/seqid=0 2024-11-08T09:25:34,187 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057934186"}]},"ts":"1731057934186"} 2024-11-08T09:25:34,188 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T09:25:34,188 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T09:25:34,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T09:25:34,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742376_1552 (size=12454) 2024-11-08T09:25:34,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, UNASSIGN}] 2024-11-08T09:25:34,190 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, UNASSIGN 2024-11-08T09:25:34,190 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=CLOSING, regionLocation=cf4d76213909,39151,1731057746690 2024-11-08T09:25:34,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, UNASSIGN because future has completed 2024-11-08T09:25:34,192 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T09:25:34,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690}] 2024-11-08T09:25:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T09:25:34,344 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:34,344 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T09:25:34,344 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing 8144a32b044f8a5dee9d583d8360a12c, disabling compactions & flushes 2024-11-08T09:25:34,344 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T09:25:34,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:34,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108614ac0c28ca4485b8f731b80ef762a97_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108614ac0c28ca4485b8f731b80ef762a97_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:34,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/2be697ef527e4da2813f57d38f0b36b9, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:34,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/2be697ef527e4da2813f57d38f0b36b9 is 175, key is test_row_0/A:col10/1731057932020/Put/seqid=0 2024-11-08T09:25:34,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742377_1553 (size=31255) 2024-11-08T09:25:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T09:25:35,001 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/2be697ef527e4da2813f57d38f0b36b9 2024-11-08T09:25:35,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/6fb368eaf054470e862de6248c9907eb is 50, key is test_row_0/B:col10/1731057932020/Put/seqid=0 2024-11-08T09:25:35,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742378_1554 (size=12301) 2024-11-08T09:25:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T09:25:35,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/6fb368eaf054470e862de6248c9907eb 2024-11-08T09:25:35,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0b8e1738fef44811bab26c3cae40b906 is 50, key is test_row_0/C:col10/1731057932020/Put/seqid=0 2024-11-08T09:25:35,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742379_1555 (size=12301) 2024-11-08T09:25:35,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0b8e1738fef44811bab26c3cae40b906 2024-11-08T09:25:35,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/2be697ef527e4da2813f57d38f0b36b9 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/2be697ef527e4da2813f57d38f0b36b9 2024-11-08T09:25:35,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/2be697ef527e4da2813f57d38f0b36b9, entries=150, sequenceid=290, filesize=30.5 K 2024-11-08T09:25:35,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/6fb368eaf054470e862de6248c9907eb as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6fb368eaf054470e862de6248c9907eb 2024-11-08T09:25:35,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6fb368eaf054470e862de6248c9907eb, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T09:25:35,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/0b8e1738fef44811bab26c3cae40b906 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0b8e1738fef44811bab26c3cae40b906 2024-11-08T09:25:35,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0b8e1738fef44811bab26c3cae40b906, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T09:25:35,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=20.13 KB/20610 for 8144a32b044f8a5dee9d583d8360a12c in 1693ms, sequenceid=290, compaction requested=true 2024-11-08T09:25:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8144a32b044f8a5dee9d583d8360a12c: 2024-11-08T09:25:35,872 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T09:25:35,872 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:35,872 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. because compaction request was cancelled 2024-11-08T09:25:35,872 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. after waiting 0 ms 2024-11-08T09:25:35,872 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:25:35,872 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:A 2024-11-08T09:25:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:35,872 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. because compaction request was cancelled 2024-11-08T09:25:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8144a32b044f8a5dee9d583d8360a12c:C, priority=-2147483648, current under compaction store size is 2 2024-11-08T09:25:35,872 DEBUG [RS:0;cf4d76213909:39151-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:B 2024-11-08T09:25:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T09:25:35,872 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing 8144a32b044f8a5dee9d583d8360a12c 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T09:25:35,872 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. because compaction request was cancelled 2024-11-08T09:25:35,872 DEBUG [RS:0;cf4d76213909:39151-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8144a32b044f8a5dee9d583d8360a12c:C 2024-11-08T09:25:35,879 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082721c12f53ba4ac2b6913367cac2a076_8144a32b044f8a5dee9d583d8360a12c is 50, key is test_row_0/A:col10/1731057934182/Put/seqid=0 2024-11-08T09:25:35,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742380_1556 (size=9914) 2024-11-08T09:25:36,287 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T09:25:36,294 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082721c12f53ba4ac2b6913367cac2a076_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082721c12f53ba4ac2b6913367cac2a076_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:36,296 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/d592b358d54a470b8719c1cb7f329ae2, store: [table=TestAcidGuarantees family=A region=8144a32b044f8a5dee9d583d8360a12c] 2024-11-08T09:25:36,297 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/d592b358d54a470b8719c1cb7f329ae2 is 175, key is test_row_0/A:col10/1731057934182/Put/seqid=0 2024-11-08T09:25:36,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742381_1557 (size=22561) 2024-11-08T09:25:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T09:25:36,704 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/d592b358d54a470b8719c1cb7f329ae2 2024-11-08T09:25:36,717 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/1b36888b196f472c9faad623a33e06c2 is 50, key is test_row_0/B:col10/1731057934182/Put/seqid=0 2024-11-08T09:25:36,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742382_1558 (size=9857) 2024-11-08T09:25:37,123 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/1b36888b196f472c9faad623a33e06c2 2024-11-08T09:25:37,137 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/1ae55171568446c0a717058490297534 is 50, key is test_row_0/C:col10/1731057934182/Put/seqid=0 2024-11-08T09:25:37,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742383_1559 (size=9857) 2024-11-08T09:25:37,545 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/1ae55171568446c0a717058490297534 2024-11-08T09:25:37,607 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/A/d592b358d54a470b8719c1cb7f329ae2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/d592b358d54a470b8719c1cb7f329ae2 2024-11-08T09:25:37,613 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/d592b358d54a470b8719c1cb7f329ae2, entries=100, sequenceid=296, filesize=22.0 K 2024-11-08T09:25:37,613 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/B/1b36888b196f472c9faad623a33e06c2 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/1b36888b196f472c9faad623a33e06c2 2024-11-08T09:25:37,617 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/1b36888b196f472c9faad623a33e06c2, entries=100, sequenceid=296, filesize=9.6 K 2024-11-08T09:25:37,618 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/.tmp/C/1ae55171568446c0a717058490297534 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1ae55171568446c0a717058490297534 2024-11-08T09:25:37,621 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1ae55171568446c0a717058490297534, entries=100, sequenceid=296, filesize=9.6 K 2024-11-08T09:25:37,622 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 8144a32b044f8a5dee9d583d8360a12c in 1750ms, sequenceid=296, compaction requested=true 2024-11-08T09:25:37,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f308e6bdae8e4a4799594fa8ffb2d827, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/e540f93193b84b20837f37edadc7a361, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/69a789fbc2384271868b705c5d0f9650, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ff818e3e5cf41da8c098de9bdf1ee73, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ccbe47603834a1886139b65ec823cd2, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/969fceb13494435fb3e543033d87bcad, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/3e7cfb57419c40a796d6b9f9c28a8e17, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/be7ae62b093042a286b3d87a5189ced0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7e6851c123746268547c35fa6c9f46a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1ef9d49baf4341ab8d07b0188d5cc122, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f91d9125ed8a477c980567bb54d1b04d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/ddc6e0e647e241c184c647b8356d486c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c3d2aff571ac4d23852e04531597c62f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1469d86f5c404d2bb4828f7b4ee7e508, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/b701aac3be7a4142bf243f6c4a9736b0, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f0b61d85219242589a051b0791c8dafd] to archive 2024-11-08T09:25:37,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:25:37,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f308e6bdae8e4a4799594fa8ffb2d827 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f308e6bdae8e4a4799594fa8ffb2d827 2024-11-08T09:25:37,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/e540f93193b84b20837f37edadc7a361 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/e540f93193b84b20837f37edadc7a361 2024-11-08T09:25:37,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/69a789fbc2384271868b705c5d0f9650 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/69a789fbc2384271868b705c5d0f9650 2024-11-08T09:25:37,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ff818e3e5cf41da8c098de9bdf1ee73 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ff818e3e5cf41da8c098de9bdf1ee73 2024-11-08T09:25:37,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ccbe47603834a1886139b65ec823cd2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/9ccbe47603834a1886139b65ec823cd2 2024-11-08T09:25:37,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/969fceb13494435fb3e543033d87bcad to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/969fceb13494435fb3e543033d87bcad 2024-11-08T09:25:37,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/3e7cfb57419c40a796d6b9f9c28a8e17 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/3e7cfb57419c40a796d6b9f9c28a8e17 2024-11-08T09:25:37,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/be7ae62b093042a286b3d87a5189ced0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/be7ae62b093042a286b3d87a5189ced0 2024-11-08T09:25:37,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7e6851c123746268547c35fa6c9f46a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7e6851c123746268547c35fa6c9f46a 2024-11-08T09:25:37,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1ef9d49baf4341ab8d07b0188d5cc122 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1ef9d49baf4341ab8d07b0188d5cc122 2024-11-08T09:25:37,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f91d9125ed8a477c980567bb54d1b04d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f91d9125ed8a477c980567bb54d1b04d 2024-11-08T09:25:37,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/ddc6e0e647e241c184c647b8356d486c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/ddc6e0e647e241c184c647b8356d486c 2024-11-08T09:25:37,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c3d2aff571ac4d23852e04531597c62f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c3d2aff571ac4d23852e04531597c62f 2024-11-08T09:25:37,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1469d86f5c404d2bb4828f7b4ee7e508 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/1469d86f5c404d2bb4828f7b4ee7e508 2024-11-08T09:25:37,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/b701aac3be7a4142bf243f6c4a9736b0 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/b701aac3be7a4142bf243f6c4a9736b0 2024-11-08T09:25:37,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f0b61d85219242589a051b0791c8dafd to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/f0b61d85219242589a051b0791c8dafd 2024-11-08T09:25:37,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/11d02dd39c8e4fa4a8b8a87317d30111, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/475aec0dd0c44c3186cce5d25313babc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/eebd6941daf04c208af3b7121eb2cef3, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/9671b1893f8c4f81af18dc970c75a15a, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/67e2352ced37455db36f5042db122d42, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/23e30aadf556429296b1537e5efaecba, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/bcaec9e1846e484eb564f3823cd9679c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/635da8b089f94bf98e06cc5deedc1880, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/dcdcf79d153149b99e0bcc1da838b104, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/f895539e661a4736a33b179168bfd83c, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/5c3b612e6ad1480e956898cb9b3ff473, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/57bc4e4982784b198d64b499108e140b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/676ad635740941a999df4de3c8df6cc6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6ba90239c1314177b9b2d3ef66ba926e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/2b86a87c698649ba96aa8825795f522d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/591afe19d00c4500aff79004d4620471] to archive 2024-11-08T09:25:37,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:25:37,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/11d02dd39c8e4fa4a8b8a87317d30111 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/11d02dd39c8e4fa4a8b8a87317d30111 2024-11-08T09:25:37,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/475aec0dd0c44c3186cce5d25313babc to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/475aec0dd0c44c3186cce5d25313babc 2024-11-08T09:25:37,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/eebd6941daf04c208af3b7121eb2cef3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/eebd6941daf04c208af3b7121eb2cef3 2024-11-08T09:25:37,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/9671b1893f8c4f81af18dc970c75a15a to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/9671b1893f8c4f81af18dc970c75a15a 2024-11-08T09:25:37,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/67e2352ced37455db36f5042db122d42 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/67e2352ced37455db36f5042db122d42 2024-11-08T09:25:37,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/23e30aadf556429296b1537e5efaecba to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/23e30aadf556429296b1537e5efaecba 2024-11-08T09:25:37,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/bcaec9e1846e484eb564f3823cd9679c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/bcaec9e1846e484eb564f3823cd9679c 2024-11-08T09:25:37,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/635da8b089f94bf98e06cc5deedc1880 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/635da8b089f94bf98e06cc5deedc1880 2024-11-08T09:25:37,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/dcdcf79d153149b99e0bcc1da838b104 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/dcdcf79d153149b99e0bcc1da838b104 2024-11-08T09:25:37,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/f895539e661a4736a33b179168bfd83c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/f895539e661a4736a33b179168bfd83c 2024-11-08T09:25:37,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/5c3b612e6ad1480e956898cb9b3ff473 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/5c3b612e6ad1480e956898cb9b3ff473 2024-11-08T09:25:37,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/57bc4e4982784b198d64b499108e140b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/57bc4e4982784b198d64b499108e140b 2024-11-08T09:25:37,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/676ad635740941a999df4de3c8df6cc6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/676ad635740941a999df4de3c8df6cc6 2024-11-08T09:25:37,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6ba90239c1314177b9b2d3ef66ba926e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6ba90239c1314177b9b2d3ef66ba926e 2024-11-08T09:25:37,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/2b86a87c698649ba96aa8825795f522d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/2b86a87c698649ba96aa8825795f522d 2024-11-08T09:25:37,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/591afe19d00c4500aff79004d4620471 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/591afe19d00c4500aff79004d4620471 2024-11-08T09:25:37,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/457e5ae6a9a3499c9efba3e80b90b8af, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0d31ee44264d4e9c9a5b2b84b45f693e, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/00ee38fd5a4d429db03b69a3af396267, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/191d8d21e26443b09b89fb95ed80f26f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/659a0828080c4ed5a82808c0aa80de07, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1c4c0eb6101f475fbee8b40ce560f9d6, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/f51d70e1402a4d8985d6e18a6f58b979, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/9b7e9042355446218cf7986d62e4bf6b, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/138dff9746e5427794cd9bd81076bce4, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e6319fae0cd14b588eb52168afb7b7fc, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/edc2aefd137840f2a1944565b0ec1282, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/05ac276b30a04bf186f224fdac39c68d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/af83eddfcf5f4bebbbca6b1877972c27, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e8b94692b3114f6ba03c067c2111dd6f, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/5c382527c71046738c2d7ba56a98185d, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/b674d029570b4e0ebd24902ed2556115] to archive 2024-11-08T09:25:37,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T09:25:37,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/457e5ae6a9a3499c9efba3e80b90b8af to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/457e5ae6a9a3499c9efba3e80b90b8af 2024-11-08T09:25:37,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0d31ee44264d4e9c9a5b2b84b45f693e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0d31ee44264d4e9c9a5b2b84b45f693e 2024-11-08T09:25:37,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/00ee38fd5a4d429db03b69a3af396267 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/00ee38fd5a4d429db03b69a3af396267 2024-11-08T09:25:37,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/191d8d21e26443b09b89fb95ed80f26f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/191d8d21e26443b09b89fb95ed80f26f 2024-11-08T09:25:37,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/659a0828080c4ed5a82808c0aa80de07 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/659a0828080c4ed5a82808c0aa80de07 2024-11-08T09:25:37,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1c4c0eb6101f475fbee8b40ce560f9d6 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1c4c0eb6101f475fbee8b40ce560f9d6 2024-11-08T09:25:37,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/f51d70e1402a4d8985d6e18a6f58b979 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/f51d70e1402a4d8985d6e18a6f58b979 2024-11-08T09:25:37,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/9b7e9042355446218cf7986d62e4bf6b to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/9b7e9042355446218cf7986d62e4bf6b 2024-11-08T09:25:37,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/138dff9746e5427794cd9bd81076bce4 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/138dff9746e5427794cd9bd81076bce4 2024-11-08T09:25:37,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e6319fae0cd14b588eb52168afb7b7fc to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e6319fae0cd14b588eb52168afb7b7fc 2024-11-08T09:25:37,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/edc2aefd137840f2a1944565b0ec1282 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/edc2aefd137840f2a1944565b0ec1282 2024-11-08T09:25:37,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/05ac276b30a04bf186f224fdac39c68d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/05ac276b30a04bf186f224fdac39c68d 2024-11-08T09:25:37,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/af83eddfcf5f4bebbbca6b1877972c27 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/af83eddfcf5f4bebbbca6b1877972c27 2024-11-08T09:25:37,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e8b94692b3114f6ba03c067c2111dd6f to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/e8b94692b3114f6ba03c067c2111dd6f 2024-11-08T09:25:37,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/5c382527c71046738c2d7ba56a98185d to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/5c382527c71046738c2d7ba56a98185d 2024-11-08T09:25:37,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/b674d029570b4e0ebd24902ed2556115 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/b674d029570b4e0ebd24902ed2556115 2024-11-08T09:25:37,678 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/recovered.edits/299.seqid, newMaxSeqId=299, maxSeqId=4 2024-11-08T09:25:37,679 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. 2024-11-08T09:25:37,679 DEBUG [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for 8144a32b044f8a5dee9d583d8360a12c: Waiting for close lock at 1731057934344Running coprocessor pre-close hooks at 1731057934344Disabling compacts and flushes for region at 1731057934344Disabling writes for close at 1731057935872 (+1528 ms)Obtaining lock to block concurrent updates at 1731057935872Preparing flush snapshotting stores in 8144a32b044f8a5dee9d583d8360a12c at 1731057935872Finished memstore snapshotting TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731057935872Flushing stores of TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c. at 1731057935873 (+1 ms)Flushing 8144a32b044f8a5dee9d583d8360a12c/A: creating writer at 1731057935873Flushing mob file 8144a32b044f8a5dee9d583d8360a12c/A: appending metadata at 1731057935879 (+6 ms)Flushing mob file 8144a32b044f8a5dee9d583d8360a12c/A: closing flushed file at 1731057935879Flushing 8144a32b044f8a5dee9d583d8360a12c/A: appending metadata at 1731057936296 (+417 ms)Flushing 8144a32b044f8a5dee9d583d8360a12c/A: closing flushed file at 1731057936296Flushing 8144a32b044f8a5dee9d583d8360a12c/B: creating writer at 1731057936712 (+416 ms)Flushing 8144a32b044f8a5dee9d583d8360a12c/B: appending metadata at 1731057936716 (+4 ms)Flushing 8144a32b044f8a5dee9d583d8360a12c/B: closing flushed file at 1731057936716Flushing 8144a32b044f8a5dee9d583d8360a12c/C: creating writer at 1731057937132 (+416 ms)Flushing 8144a32b044f8a5dee9d583d8360a12c/C: appending metadata at 1731057937137 (+5 ms)Flushing 8144a32b044f8a5dee9d583d8360a12c/C: closing flushed file at 1731057937137Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@318626fb: reopening flushed file at 1731057937606 (+469 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e3a4e8a: reopening flushed file at 1731057937613 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d2b3ce0: reopening flushed file at 1731057937617 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 8144a32b044f8a5dee9d583d8360a12c in 1750ms, sequenceid=296, compaction requested=true at 1731057937622 (+5 ms)Writing region close event to WAL at 1731057937676 (+54 ms)Running coprocessor post-close hooks at 1731057937679 (+3 ms)Closed at 1731057937679 2024-11-08T09:25:37,681 INFO [RS_CLOSE_REGION-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed 8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:37,681 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=8144a32b044f8a5dee9d583d8360a12c, regionState=CLOSED 2024-11-08T09:25:37,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 because future has completed 2024-11-08T09:25:37,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-08T09:25:37,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure 8144a32b044f8a5dee9d583d8360a12c, server=cf4d76213909,39151,1731057746690 in 3.4920 sec 2024-11-08T09:25:37,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-08T09:25:37,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8144a32b044f8a5dee9d583d8360a12c, UNASSIGN in 3.4970 sec 2024-11-08T09:25:37,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-08T09:25:37,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.5010 sec 2024-11-08T09:25:37,693 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731057937693"}]},"ts":"1731057937693"} 2024-11-08T09:25:37,694 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T09:25:37,694 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T09:25:37,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.5110 sec 2024-11-08T09:25:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T09:25:38,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:38,354 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T09:25:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:38,359 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T09:25:38,361 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:38,365 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,369 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C, FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/recovered.edits] 2024-11-08T09:25:38,374 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/2be697ef527e4da2813f57d38f0b36b9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/2be697ef527e4da2813f57d38f0b36b9 2024-11-08T09:25:38,376 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/5254af955ffe4d0ba9019dc13f027e5e to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/5254af955ffe4d0ba9019dc13f027e5e 2024-11-08T09:25:38,378 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7c0c2c8e6a14801969091ea2e428464 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/c7c0c2c8e6a14801969091ea2e428464 2024-11-08T09:25:38,379 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/d592b358d54a470b8719c1cb7f329ae2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/d592b358d54a470b8719c1cb7f329ae2 2024-11-08T09:25:38,382 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/fa3234c8ec594b048a9afa437c5e36e9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/A/fa3234c8ec594b048a9afa437c5e36e9 2024-11-08T09:25:38,386 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/1b36888b196f472c9faad623a33e06c2 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/1b36888b196f472c9faad623a33e06c2 2024-11-08T09:25:38,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6fb368eaf054470e862de6248c9907eb to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/6fb368eaf054470e862de6248c9907eb 2024-11-08T09:25:38,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/74d1654e7f784fca8aeecacb013c8dde to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/74d1654e7f784fca8aeecacb013c8dde 2024-11-08T09:25:38,390 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/80cd97a5e1104568867b23293040ac47 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/80cd97a5e1104568867b23293040ac47 2024-11-08T09:25:38,391 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/85ffe52edea54a5aaa2be201b4e424d9 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/B/85ffe52edea54a5aaa2be201b4e424d9 2024-11-08T09:25:38,392 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0b8e1738fef44811bab26c3cae40b906 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0b8e1738fef44811bab26c3cae40b906 2024-11-08T09:25:38,393 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0cecc5d08d0848308b30293d1826d7d3 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/0cecc5d08d0848308b30293d1826d7d3 2024-11-08T09:25:38,394 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1ae55171568446c0a717058490297534 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/1ae55171568446c0a717058490297534 2024-11-08T09:25:38,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/799dd11f2f20422981e2ce2e8ef6a0a1 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/799dd11f2f20422981e2ce2e8ef6a0a1 2024-11-08T09:25:38,396 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/93f496776b044944b5f29091f8ef7ba7 to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/C/93f496776b044944b5f29091f8ef7ba7 2024-11-08T09:25:38,399 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/recovered.edits/299.seqid to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c/recovered.edits/299.seqid 2024-11-08T09:25:38,399 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/default/TestAcidGuarantees/8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,399 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T09:25:38,400 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T09:25:38,401 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T09:25:38,404 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082721c12f53ba4ac2b6913367cac2a076_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082721c12f53ba4ac2b6913367cac2a076_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,405 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082b156d4947f7467c97e80e4acd1befc5_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082b156d4947f7467c97e80e4acd1befc5_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,407 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083a4dc26e263a4fe998d32e4ae289942d_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083a4dc26e263a4fe998d32e4ae289942d_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,408 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108614ac0c28ca4485b8f731b80ef762a97_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108614ac0c28ca4485b8f731b80ef762a97_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,409 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110861d53da8c52a4adba27e4c0f477eeeca_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110861d53da8c52a4adba27e4c0f477eeeca_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,410 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108628b3086071a4e95833b3a787fcc07f4_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108628b3086071a4e95833b3a787fcc07f4_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,411 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108631cbe61c1d8460da5272ed50dadfcbe_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108631cbe61c1d8460da5272ed50dadfcbe_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,412 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088f015a62855545e79729924e1449dfb2_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088f015a62855545e79729924e1449dfb2_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,413 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110899c6e6c1254647d08186a4fa1d9580b2_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110899c6e6c1254647d08186a4fa1d9580b2_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,413 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a8cb55147714212a36094020a4b3dc2_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a8cb55147714212a36094020a4b3dc2_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,414 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a2e8ecc3799844b7a6c8ff1b6585717f_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a2e8ecc3799844b7a6c8ff1b6585717f_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,415 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b33858b8d04e407392459b5a95989263_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b33858b8d04e407392459b5a95989263_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,415 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b65d082fe4a94cc2a859fd20c16b28ad_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b65d082fe4a94cc2a859fd20c16b28ad_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,416 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c1c069b173bf48d6b54024cc98bc4105_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c1c069b173bf48d6b54024cc98bc4105_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,417 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108deb9883977774684a24b79b0a2ea293f_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108deb9883977774684a24b79b0a2ea293f_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,418 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f11d3f0112ac484e9bc977310bd0b4e3_8144a32b044f8a5dee9d583d8360a12c to hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f11d3f0112ac484e9bc977310bd0b4e3_8144a32b044f8a5dee9d583d8360a12c 2024-11-08T09:25:38,418 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T09:25:38,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:38,422 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T09:25:38,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T09:25:38,424 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:38,424 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T09:25:38,424 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731057938424"}]},"ts":"9223372036854775807"} 2024-11-08T09:25:38,426 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T09:25:38,426 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8144a32b044f8a5dee9d583d8360a12c, NAME => 'TestAcidGuarantees,,1731057909203.8144a32b044f8a5dee9d583d8360a12c.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T09:25:38,426 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T09:25:38,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731057938426"}]},"ts":"9223372036854775807"} 2024-11-08T09:25:38,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T09:25:38,427 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T09:25:38,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-11-08T09:25:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T09:25:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T09:25:38,472 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T09:25:38,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 237) - Thread LEAK? -, OpenFileDescriptor=457 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=5401 (was 5437) 2024-11-08T09:25:38,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T09:25:38,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:25:38,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:25:38,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:38,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:38,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T09:25:38,489 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T09:25:38,489 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-08T09:25:38,490 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cf4d76213909,35807,1731057745600 2024-11-08T09:25:38,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T09:25:38,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T09:25:38,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:25:38,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:25:38,565 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T09:25:38,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T09:25:38,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T09:25:38,568 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T09:25:38,568 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:25:38,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:38,569 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cf4d76213909,39151,1731057746690' ***** 2024-11-08T09:25:38,569 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T09:25:38,569 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T09:25:38,570 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(959): stopping server cf4d76213909,39151,1731057746690 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cf4d76213909:39151. 2024-11-08T09:25:38,570 DEBUG [RS:0;cf4d76213909:39151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T09:25:38,570 DEBUG [RS:0;cf4d76213909:39151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T09:25:38,570 INFO [RS:0;cf4d76213909:39151 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T09:25:38,571 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T09:25:38,571 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T09:25:38,571 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T09:25:38,572 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:38,572 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T09:25:38,572 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T09:25:38,572 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T09:25:38,572 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T09:25:38,572 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T09:25:38,572 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-08T09:25:38,595 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/info/a780a692bdbc45c4b9ff26ee7b476b6d is 95, key is TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb./info:/1731057778901/DeleteFamily/seqid=0 2024-11-08T09:25:38,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742384_1560 (size=6514) 2024-11-08T09:25:38,600 INFO [regionserver/cf4d76213909:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T09:25:38,600 INFO [regionserver/cf4d76213909:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T09:25:38,622 INFO [regionserver/cf4d76213909:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T09:25:38,772 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:38,972 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:39,000 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/info/a780a692bdbc45c4b9ff26ee7b476b6d 2024-11-08T09:25:39,028 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/ns/369cdeec8e4a4673b2f0e2c17b51f920 is 93, key is TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb./ns:/1731057778866/DeleteFamily/seqid=0 2024-11-08T09:25:39,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742385_1561 (size=5906) 2024-11-08T09:25:39,173 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:39,373 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:39,433 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/ns/369cdeec8e4a4673b2f0e2c17b51f920 2024-11-08T09:25:39,458 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/rep_barrier/07c1adfbfea54b7698e30f7c8f288f1a is 102, key is TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb./rep_barrier:/1731057778866/DeleteFamily/seqid=0 2024-11-08T09:25:39,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742386_1562 (size=6025) 2024-11-08T09:25:39,573 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:39,774 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:39,862 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/rep_barrier/07c1adfbfea54b7698e30f7c8f288f1a 2024-11-08T09:25:39,885 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/table/b7eb7c8b90b548bf9597b9421ae6d3e4 is 96, key is TestAcidGuarantees,,1731057750638.6f50fcf560ee3a7bd187b06bd37f77bb./table:/1731057778866/DeleteFamily/seqid=0 2024-11-08T09:25:39,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742387_1563 (size=5936) 2024-11-08T09:25:39,974 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:40,174 DEBUG [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T09:25:40,290 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/table/b7eb7c8b90b548bf9597b9421ae6d3e4 2024-11-08T09:25:40,300 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/info/a780a692bdbc45c4b9ff26ee7b476b6d as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/info/a780a692bdbc45c4b9ff26ee7b476b6d 2024-11-08T09:25:40,304 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/info/a780a692bdbc45c4b9ff26ee7b476b6d, entries=12, sequenceid=90, filesize=6.4 K 2024-11-08T09:25:40,305 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/ns/369cdeec8e4a4673b2f0e2c17b51f920 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/ns/369cdeec8e4a4673b2f0e2c17b51f920 2024-11-08T09:25:40,309 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/ns/369cdeec8e4a4673b2f0e2c17b51f920, entries=8, sequenceid=90, filesize=5.8 K 2024-11-08T09:25:40,310 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/rep_barrier/07c1adfbfea54b7698e30f7c8f288f1a as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/rep_barrier/07c1adfbfea54b7698e30f7c8f288f1a 2024-11-08T09:25:40,314 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/rep_barrier/07c1adfbfea54b7698e30f7c8f288f1a, entries=6, sequenceid=90, filesize=5.9 K 2024-11-08T09:25:40,314 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/.tmp/table/b7eb7c8b90b548bf9597b9421ae6d3e4 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/table/b7eb7c8b90b548bf9597b9421ae6d3e4 2024-11-08T09:25:40,317 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/table/b7eb7c8b90b548bf9597b9421ae6d3e4, entries=7, sequenceid=90, filesize=5.8 K 2024-11-08T09:25:40,318 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false 2024-11-08T09:25:40,322 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-08T09:25:40,323 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T09:25:40,323 INFO [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T09:25:40,323 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731057938571Running coprocessor pre-close hooks at 1731057938572 (+1 ms)Disabling compacts and flushes for region at 1731057938572Disabling writes for close at 1731057938572Obtaining lock to block concurrent updates at 1731057938572Preparing flush snapshotting stores in 1588230740 at 1731057938572Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731057938572Flushing stores of hbase:meta,,1.1588230740 at 1731057938573 (+1 ms)Flushing 1588230740/info: creating writer at 1731057938573Flushing 1588230740/info: appending metadata at 1731057938595 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731057938595Flushing 1588230740/ns: creating writer at 1731057939010 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731057939028 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731057939028Flushing 1588230740/rep_barrier: creating writer at 1731057939441 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731057939458 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731057939458Flushing 1588230740/table: creating writer at 1731057939867 (+409 ms)Flushing 1588230740/table: appending metadata at 1731057939885 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731057939885Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c932773: reopening flushed file at 1731057940299 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28da5383: reopening flushed file at 1731057940305 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eddc42c: reopening flushed file at 1731057940309 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cf13aef: reopening flushed file at 1731057940314 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false at 1731057940318 (+4 ms)Writing region close event to WAL at 1731057940320 (+2 ms)Running coprocessor post-close hooks at 1731057940323 (+3 ms)Closed at 1731057940323 2024-11-08T09:25:40,323 DEBUG [RS_CLOSE_META-regionserver/cf4d76213909:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T09:25:40,363 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=cf4d76213909:35807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-08T09:25:40,375 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(976): stopping server cf4d76213909,39151,1731057746690; all regions closed. 2024-11-08T09:25:40,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741834_1010 (size=25344) 2024-11-08T09:25:40,385 DEBUG [RS:0;cf4d76213909:39151 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/oldWALs 2024-11-08T09:25:40,385 INFO [RS:0;cf4d76213909:39151 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cf4d76213909%2C39151%2C1731057746690.meta:.meta(num 1731057750130) 2024-11-08T09:25:40,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741833_1009 (size=12547694) 2024-11-08T09:25:40,390 DEBUG [RS:0;cf4d76213909:39151 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/oldWALs 2024-11-08T09:25:40,390 INFO [RS:0;cf4d76213909:39151 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cf4d76213909%2C39151%2C1731057746690:(num 1731057749745) 2024-11-08T09:25:40,390 DEBUG [RS:0;cf4d76213909:39151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T09:25:40,390 INFO [RS:0;cf4d76213909:39151 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T09:25:40,390 INFO [RS:0;cf4d76213909:39151 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T09:25:40,391 INFO [RS:0;cf4d76213909:39151 {}] hbase.ChoreService(370): Chore service for: regionserver/cf4d76213909:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T09:25:40,391 INFO [RS:0;cf4d76213909:39151 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T09:25:40,391 INFO [regionserver/cf4d76213909:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T09:25:40,392 INFO [RS:0;cf4d76213909:39151 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39151 2024-11-08T09:25:40,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cf4d76213909,39151,1731057746690 2024-11-08T09:25:40,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T09:25:40,458 INFO [RS:0;cf4d76213909:39151 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T09:25:40,460 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cf4d76213909,39151,1731057746690] 2024-11-08T09:25:40,479 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cf4d76213909,39151,1731057746690 already deleted, retry=false 2024-11-08T09:25:40,480 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cf4d76213909,39151,1731057746690 expired; onlineServers=0 2024-11-08T09:25:40,480 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cf4d76213909,35807,1731057745600' ***** 2024-11-08T09:25:40,480 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T09:25:40,480 INFO [M:0;cf4d76213909:35807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T09:25:40,480 INFO [M:0;cf4d76213909:35807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T09:25:40,480 DEBUG [M:0;cf4d76213909:35807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T09:25:40,480 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T09:25:40,480 DEBUG [M:0;cf4d76213909:35807 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T09:25:40,480 DEBUG [master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.large.0-1731057749291 {}] cleaner.HFileCleaner(306): Exit Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.large.0-1731057749291,5,FailOnTimeoutGroup] 2024-11-08T09:25:40,480 DEBUG [master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.small.0-1731057749292 {}] cleaner.HFileCleaner(306): Exit Thread[master/cf4d76213909:0:becomeActiveMaster-HFileCleaner.small.0-1731057749292,5,FailOnTimeoutGroup] 2024-11-08T09:25:40,481 INFO [M:0;cf4d76213909:35807 {}] hbase.ChoreService(370): Chore service for: master/cf4d76213909:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T09:25:40,481 INFO [M:0;cf4d76213909:35807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T09:25:40,481 DEBUG [M:0;cf4d76213909:35807 {}] master.HMaster(1795): Stopping service threads 2024-11-08T09:25:40,481 INFO [M:0;cf4d76213909:35807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T09:25:40,481 INFO [M:0;cf4d76213909:35807 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T09:25:40,481 ERROR [M:0;cf4d76213909:35807 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:43921 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:43921,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-08T09:25:40,482 INFO [M:0;cf4d76213909:35807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T09:25:40,482 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T09:25:40,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T09:25:40,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T09:25:40,491 DEBUG [M:0;cf4d76213909:35807 {}] zookeeper.ZKUtil(347): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T09:25:40,491 WARN [M:0;cf4d76213909:35807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T09:25:40,491 INFO [M:0;cf4d76213909:35807 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/.lastflushedseqids 2024-11-08T09:25:40,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742388_1564 (size=72) 2024-11-08T09:25:40,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T09:25:40,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39151-0x10119c0f6eb0001, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T09:25:40,572 INFO [RS:0;cf4d76213909:39151 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T09:25:40,572 INFO [RS:0;cf4d76213909:39151 {}] regionserver.HRegionServer(1031): Exiting; stopping=cf4d76213909,39151,1731057746690; zookeeper connection closed. 2024-11-08T09:25:40,573 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e0a9886 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e0a9886 2024-11-08T09:25:40,574 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T09:25:40,907 INFO [M:0;cf4d76213909:35807 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T09:25:40,907 INFO [M:0;cf4d76213909:35807 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T09:25:40,907 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T09:25:40,907 INFO [M:0;cf4d76213909:35807 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:25:40,908 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:25:40,908 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T09:25:40,908 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:25:40,908 INFO [M:0;cf4d76213909:35807 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=728.53 KB heapSize=893.87 KB 2024-11-08T09:25:40,929 DEBUG [M:0;cf4d76213909:35807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba15e726ac034c7497d6f7a62f596781 is 82, key is hbase:meta,,1/info:regioninfo/1731057750215/Put/seqid=0 2024-11-08T09:25:40,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742389_1565 (size=5672) 2024-11-08T09:25:41,333 INFO [M:0;cf4d76213909:35807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2039 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba15e726ac034c7497d6f7a62f596781 2024-11-08T09:25:41,365 DEBUG [M:0;cf4d76213909:35807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c49474e480324666a5613d5599da4a28 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x1F/proc:d/1731057782132/Put/seqid=0 2024-11-08T09:25:41,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742390_1566 (size=42379) 2024-11-08T09:25:41,770 INFO [M:0;cf4d76213909:35807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=727.97 KB at sequenceid=2039 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c49474e480324666a5613d5599da4a28 2024-11-08T09:25:41,779 INFO [M:0;cf4d76213909:35807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c49474e480324666a5613d5599da4a28 2024-11-08T09:25:41,799 DEBUG [M:0;cf4d76213909:35807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/50d1391962da4485b1472aba267cd899 is 69, key is cf4d76213909,39151,1731057746690/rs:state/1731057749451/Put/seqid=0 2024-11-08T09:25:41,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073742391_1567 (size=5156) 2024-11-08T09:25:42,204 INFO [M:0;cf4d76213909:35807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2039 (bloomFilter=true), to=hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/50d1391962da4485b1472aba267cd899 2024-11-08T09:25:42,214 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba15e726ac034c7497d6f7a62f596781 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba15e726ac034c7497d6f7a62f596781 2024-11-08T09:25:42,218 INFO [M:0;cf4d76213909:35807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba15e726ac034c7497d6f7a62f596781, entries=8, sequenceid=2039, filesize=5.5 K 2024-11-08T09:25:42,218 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c49474e480324666a5613d5599da4a28 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c49474e480324666a5613d5599da4a28 2024-11-08T09:25:42,221 INFO [M:0;cf4d76213909:35807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c49474e480324666a5613d5599da4a28 2024-11-08T09:25:42,221 INFO [M:0;cf4d76213909:35807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c49474e480324666a5613d5599da4a28, entries=160, sequenceid=2039, filesize=41.4 K 2024-11-08T09:25:42,222 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/50d1391962da4485b1472aba267cd899 as hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/50d1391962da4485b1472aba267cd899 2024-11-08T09:25:42,225 INFO [M:0;cf4d76213909:35807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43921/user/jenkins/test-data/14e13bc7-10c3-c6b9-febe-0919612b65b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/50d1391962da4485b1472aba267cd899, entries=1, sequenceid=2039, filesize=5.0 K 2024-11-08T09:25:42,226 INFO [M:0;cf4d76213909:35807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~728.53 KB/746013, heapSize ~893.57 KB/915016, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=2039, compaction requested=false 2024-11-08T09:25:42,227 INFO [M:0;cf4d76213909:35807 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T09:25:42,227 DEBUG [M:0;cf4d76213909:35807 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731057940907Disabling compacts and flushes for region at 1731057940907Disabling writes for close at 1731057940908 (+1 ms)Obtaining lock to block concurrent updates at 1731057940908Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731057940908Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=746013, getHeapSize=915256, getOffHeapSize=0, getCellsCount=2224 at 1731057940909 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731057940911 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731057940911Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731057940928 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731057940928Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731057941343 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731057941365 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731057941365Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731057941779 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731057941798 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731057941798Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a40cb42: reopening flushed file at 1731057942214 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3480a0b4: reopening flushed file at 1731057942218 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72c64731: reopening flushed file at 1731057942221 (+3 ms)Finished flush of dataSize ~728.53 KB/746013, heapSize ~893.57 KB/915016, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=2039, compaction requested=false at 1731057942226 (+5 ms)Writing region close event to WAL at 1731057942227 (+1 ms)Closed at 1731057942227 2024-11-08T09:25:42,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34747 is added to blk_1073741830_1006 (size=879707) 2024-11-08T09:25:42,230 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T09:25:42,230 INFO [M:0;cf4d76213909:35807 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T09:25:42,230 INFO [M:0;cf4d76213909:35807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35807 2024-11-08T09:25:42,231 INFO [M:0;cf4d76213909:35807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T09:25:42,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T09:25:42,400 INFO [M:0;cf4d76213909:35807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T09:25:42,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x10119c0f6eb0000, quorum=127.0.0.1:57608, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T09:25:42,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T09:25:42,447 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T09:25:42,447 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T09:25:42,447 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T09:25:42,448 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/hadoop.log.dir/,STOPPED} 2024-11-08T09:25:42,452 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T09:25:42,452 WARN [BP-1227818152-172.17.0.2-1731057741605 heartbeating to localhost/127.0.0.1:43921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T09:25:42,452 WARN [BP-1227818152-172.17.0.2-1731057741605 heartbeating to localhost/127.0.0.1:43921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1227818152-172.17.0.2-1731057741605 (Datanode Uuid 3f329c0b-3c7d-416d-976d-4b847f832cf6) service to localhost/127.0.0.1:43921 2024-11-08T09:25:42,452 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T09:25:42,456 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f/data/data1/current/BP-1227818152-172.17.0.2-1731057741605 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T09:25:42,456 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/cluster_36ef2d14-852b-1443-6dbc-36c5fd609a8f/data/data2/current/BP-1227818152-172.17.0.2-1731057741605 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T09:25:42,457 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T09:25:42,463 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T09:25:42,464 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T09:25:42,464 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T09:25:42,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T09:25:42,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bcca8ded-4c9e-a608-45a5-014637e688ec/hadoop.log.dir/,STOPPED} 2024-11-08T09:25:42,477 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T09:25:42,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down